waiting for #3682

2 files changed, 194 insertions, 65 deletions

diff --git a/fuzzer.py b/fuzzer.py
new file mode 100755
index 0000000..b1daf9b
--- /dev/null
+++ b/fuzzer.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import functools
+import random
+from multiprocessing.pool import ThreadPool as Pool
+
+import requests
+from pyinfluxtools import *
+from influxdb import InfluxDBClient
+
+settings = {
+    'host': 'localhost',
+    'port': 8086,
+    'username': 'root',
+    'password': 'root',
+    'database': 'test',
+}
+
+write_url = ("http://{host}:{port}/write?db={database}" +
+             "&username={username}&password={password}"
+             ).format(**settings)
+
+client = InfluxDBClient(settings['host'], settings['port'],
+                        settings['username'], settings['password'],
+                        settings['database'])
+
+
+class Generator:
+    all = lambda x: bytes(chr(random.randint(0x00, 0xff)), 'latin-1')
+    _printables = list(map(chr, list(range(0x20, 0x7E))))
+    printable = lambda x: random.choice(Generator._printables)
+    numericText = lambda x: chr(random.randint(ord("0"), ord("9")))
+    text = lambda x: chr(random.choice(
+        list(range(ord("a"), ord("z"))) +
+        list(range(ord("A"), ord("Z")))))
+
+
+class Filter:
+    regex = lambda r: re.compile(r).match
+    alpha = re.compile("^[a-z]+$").match
+    pass_all = lambda _: True
+    @staticmethod
+    def except_chars(chars):
+        regex = re.compile("^[^"+chars+"]*$")
+        return regex.match
+            
+
+
+def generate(length_min, length_max, filter, generator):
+    length = random.randint(length_min, length_max)
+    while True:
+        text = functools.reduce(lambda a,b: a+b,map(generator, range(length)))
+        if filter(text):
+            return text
+
+
+def run(*a):
+    while True:
+        test1 = generate(3, 6, 
+                Filter.regex("^[^\{\"]"), # influxdb doesn't allow measurements starting with { or "
+                Generator.printable)
+        test2 = generate(3, 6, Filter.pass_all, Generator.printable)
+        test3 = generate(3, 6, Filter.pass_all, Generator.printable)
+        test4 = generate(3, 6, Filter.pass_all, Generator.printable)
+        w = Write(test1,  # key
+                  {test2: test3},  # tags
+                  {"value": test4})  # values
+        line = str(w)
+        print(line)
+        try:
+            r = requests.post(write_url, line)
+            assert r.status_code == 204, \
+                "Data:\nline={}\nwrite={}\nr.status_code={}\nr.content={}".format(
+                    line, repr(w), r.status_code, r.text)
+        except Exception as e:
+            print(e)
+            assert False, "Data\nline={}\nwrite={}".format(str(w), repr(w))
+
+        measurement = Write.escape_value(w.key)
+        query = "SELECT * FROM {measurement} WHERE time >= now() - 1s".format(**locals())
+        result = list(client.query(query))
+        DEBUGINFO = "DEBUG:\nquery={query}\nresult={result}\nline={line}".format(**locals())
+        assert len(result) == 1, DEBUGINFO
+        assert result[0][0]['value'] == test4, DEBUGINFO + "\nvalue=" + result[0][0]['value'] + "\ntest4=" + test4
+
+
+N_PROC = 1
+with Pool(processes=N_PROC) as pool:
+    try:
+        for res in pool.imap_unordered(run, [None] * N_PROC):
+            pool.terminate()
+            print("Exit")
+            raise SystemExit(0)
+    except Exception as e:
+        pool.terminate()
+        raise e
diff --git a/pyinfluxtools/__init__.py b/pyinfluxtools/__init__.py
index 543b7b2..68146ff 100644
--- a/pyinfluxtools/__init__.py
+++ b/pyinfluxtools/__init__.py
@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 import re
 import sys
+from functools import reduce
 
 from funcparserlib.lexer import make_tokenizer
-from funcparserlib.parser import (some, maybe, many, finished, skip, NoParseError)
+from funcparserlib.parser import (
+    some, maybe, many, finished, skip, NoParseError)
 
 
 class WriteRequest(object):
@@ -22,8 +24,8 @@ class WriteRequest(object):
         >>> lines += ['cpu,host=serverA,region=us-west field1=1,field2=2 1234']
         >>> print("\\n".join(map(str, WriteRequest.parse("\\n".join(lines)))))
         cpu field=123
-        cpu,host="serverA",region="us-west" field1=1,field2=2
-        cpu,host="serverA",region="us-west" field1=1,field2=2 1234
+        cpu,host=serverA,region=us-west field1=1,field2=2
+        cpu,host=serverA,region=us-west field1=1,field2=2 1234
         """
         writes = map(Write.parse, lines.split("\n"))
         return list(writes)
@@ -64,41 +66,38 @@ class Write(object):
         """
         Parse a line from the POST request into a Write object.
 
-        >>> Write.parse('cpu a=1')
+        >>> line='cpu a=1'; Write.parse(line); print(Write.parse(line))
         <Write key=cpu tags=[] fields=[('a', 1)] timestamp=None>
-
-        >>> print(Write.parse('cpu a=1'))
         cpu a=1
 
-        >>> print(Write.parse('yahoo.CHFGBP=X.ask,tag=foobar value=10.2'))
-        yahoo.CHFGBP=X.ask,tag="foobar" value=10.2
+        >>> print(Write.parse('yahoo.CHFGBP\\=X.ask,tag=foobar value=10.2'))
+        yahoo.CHFGBP\=X.ask,tag=foobar value=10.2
 
-        >>> Write.parse('cpu,host=serverA,region=us-west foo=bar')
+        >>> Write.parse('cpu,host=serverA,region=us-west foo="bar"')
         <Write key=cpu tags=[('host', 'serverA'), ('region', 'us-west')] fields=[('foo', 'bar')] timestamp=None>
 
-        >>> print(Write.parse('cpu host=serverA,region=us-west'))
+        >>> print(Write.parse('cpu host="serverA",region="us-west"'))
         cpu host="serverA",region="us-west"
 
-        >>> Write.parse('cpu\\,01 host=serverA,region=us-west')
+        >>> line='cpu\\,01 host="serverA",region="us-west"'; \\
+        ... Write.parse(line); print(Write.parse(line))
         <Write key=cpu,01 tags=[] fields=[('host', 'serverA'), ('region', 'us-west')] timestamp=None>
-
-        >>> print(Write.parse('cpu\,01 host=serverA,region=us-west'))
         cpu\,01 host="serverA",region="us-west"
 
-        >>> Write.parse('cpu host=server\\ A,region=us\\ west')
+        >>> Write.parse('cpu host="server A",region="us west"')
         <Write key=cpu tags=[] fields=[('host', 'server A'), ('region', 'us west')] timestamp=None>
 
-        >>> Write.parse('cpu ho\\=st=server\ A,region=us\ west')
+        >>> line='cpu ho\\=st="server A",region="us west"'; \\
+        ... Write.parse(line); print(Write.parse(line))
         <Write key=cpu tags=[] fields=[('ho=st', 'server A'), ('region', 'us west')] timestamp=None>
-
-        >>> print(Write.parse('cpu ho\=st=server\ A,region=us\ west'))
         cpu ho\=st="server A",region="us west"
 
         >>> print(Write.parse('cpu,ho\=st=server\ A field=123'))
-        cpu,ho\=st="server A" field=123
+        cpu,ho\=st=server\ A field=123
 
-        >>> print(Write.parse('cpu,foo=bar,foo=bar field=123,field=123')) # error: double name is accepted
-        cpu,foo="bar",foo="bar" field=123,field=123
+        # error: double name is accepted
+        >>> print(Write.parse('cpu,foo=bar,foo=bar field=123,field=123'))
+        cpu,foo=bar,foo=bar field=123,field=123
 
         >>> print(Write.parse('cpu field12=12'))
         cpu field12=12
@@ -106,25 +105,22 @@ class Write(object):
         >>> print(Write.parse('cpu field12=12 123123123'))
         cpu field12=12 123123123
 
-        >>> try:
-        ...     print(Write.parse('cpu field12=12 1231abcdef123'))
-        ... except NoParseError:
-        ...     pass
+        >>> try: print(Write.parse('cpu field12=12 1231abcdef123'))
+        ... except NoParseError: pass
 
-        >>> print(Write.parse("cpu,x=3,y=4,z=6 field\ name=\\"HH \\\\\\"World\\",x=asdf\\\\ foo"))
+        >>> print(Write.parse('cpu,x=3,y=4,z=6 field\ name="HH \\\\\\"World",x="asdf foo"'))
         cpu,x=3,y=4,z=6 field\\ name="HH \\"World",x="asdf foo"
 
-        >>> print(Write.parse("cpu,x=3 field\ name=\\"HH \\\\\\"World\\",x=asdf\\\\ foo"))
+        >>> print(Write.parse("cpu,x=3 field\ name=\\"HH \\\\\\"World\\",x=\\"asdf foo\\""))
         cpu,x=3 field\\ name="HH \\"World",x="asdf foo"
 
-        >>> print(Write.parse("cpu foo=bar 12345"))
+        >>> print(Write.parse("cpu foo=\\"bar\\" 12345"))
         cpu foo="bar" 12345
 
-        >>> print(Write.parse('"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with field_key\\\\\\="string field value, only \\\\" need be quoted"'))
-        "measurement\ with\ quotes",tag\ key\ with\ spaces="tag,value,with" field_key\\\\="string field value, only \\" need be quoted"
-
-        >>> Write.parse('"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\\\="string field value, only \\\\" need be quoted"')
+        >>> line='"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\="string field value, only \\\\" need be quoted"'; \\
+        ... Write.parse(line); print(Write.parse(line))
         <Write key="measurement with quotes" tags=[('tag key with spaces', 'tag,value,with"commas"')] fields=[('field_key\\\\', 'string field value, only " need be quoted')] timestamp=None>
+        "measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\="string field value, only \\\" need be quoted"
 
         >>> Write.parse('disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents"')
         <Write key=disk_free tags=[] fields=[('value', 442221834240), ('working directories', 'C:\\\\My Documents\\\\Stuff for examples,C:\\\\My Documents')] timestamp=None>
@@ -132,30 +128,51 @@ class Write(object):
         >>> Write.parse('disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents" 123')
         <Write key=disk_free tags=[] fields=[('value', 442221834240), ('working directories', 'C:\\\\My Documents\\\\Stuff for examples,C:\\\\My Documents')] timestamp=123>
 
+        >>> print(Write.parse('foo,foo=2 "field key with space"="string field"'))
+        foo,foo=2 field\ key\ with\ space="string field"
+
         >>> print(Write.parse('foo,foo=2 field_key\\\\\\="string field"'))
         foo,foo=2 field_key\\\\="string field"
 
         >>> print(Write.parse('foo,foo=2 field_key="string\\\\" field"'))
         foo,foo=2 field_key="string\\" field"
 
-        >>> print(Write.parse('foo field0=tag,field1=t,field2=true,field3=True,field4=TRUE'))
+        >>> line='foo field0="tag",field1=t,field2=true,field3=True,field4=TRUE'; \\
+        ... Write.parse(line); print(Write.parse(line))
+        <Write key=foo tags=[] fields=[('field0', 'tag'), ('field1', True), ('field2', True), ('field3', True), ('field4', True)] timestamp=None>
         foo field0="tag",field1=True,field2=True,field3=True,field4=True
 
-        >>> print(Write.parse('foo field1=f,field2=false,field3=False,field4=FALSE,field5=fag'))
+        >>> line='foo field1=f,field2=false,field3=False,field4=FALSE,field5="fag"'; \\
+        ... Write.parse(line); print(Write.parse(line))
+        <Write key=foo tags=[] fields=[('field1', False), ('field2', False), ('field3', False), ('field4', False), ('field5', 'fag')] timestamp=None>
         foo field1=False,field2=False,field3=False,field4=False,field5="fag"
 
-        >>> print(Write.parse('"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\="string field value, only \\\\" need be quoted"'))
-        "measurement\ with\ quotes",tag\ key\ with\ spaces="tag,value,with\\"commas\\"" field_key\\\\="string field value, only \\" need be quoted"
+        >>> line='"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\="string field value, only \\\\" need be quoted"'; \\
+        ... Write.parse(line); print(Write.parse(line))
+        <Write key="measurement with quotes" tags=[('tag key with spaces', 'tag,value,with"commas"')] fields=[('field_key\\\\', 'string field value, only " need be quoted')] timestamp=None>
+        "measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\="string field value, only \\" need be quoted"
 
         >>> Write.parse('"measurement\ with\ quotes" foo=1')
         <Write key="measurement with quotes" tags=[] fields=[('foo', 1)] timestamp=None>
+
+        >>> print(Write.parse('K.5S,Ccpvo="eSLyE" value="7F\\\\\\\\\\\\""'))
+        K.5S,Ccpvo="eSLyE" value="7F\\\\\\\\\\\""
+
+        >>> print(Write.parse('K.5S,Ccpvo=a\\ b value=1'))
+        K.5S,Ccpvo=a\\ b value=1
+
+        >>> print(Write('test', [('a','b')], [('value','asd\\\\')]))
+        test,a=b value="asd\\\\"
         """
 
         tokval = lambda t: t.value
         joinval = "".join
         someToken = lambda type: some(lambda t: t.type == type)
-        true_values = ["t", "true", "True", "TRUE"]
-        false_values = ["f", "false", "False", "FALSE"]
+        someCharValue = lambda string: \
+            reduce(lambda a, b: a + b,
+                   map(lambda char:
+                       some(lambda t: t.value == char) >> tokval,
+                       string)) >> joinval
 
         char = someToken('Char') >> tokval
         space = someToken('Space') >> tokval
@@ -163,6 +180,10 @@ class Write(object):
         quote = someToken('Quote') >> tokval
         escape = someToken('Escape') >> tokval
         equal = someToken('Equal') >> tokval
+        true_value = (someCharValue("true") | someCharValue("t") |
+                      someCharValue("True") | someCharValue("TRUE") | someCharValue("T"))
+        false_value = (someCharValue("false") | someCharValue("f") |
+                       someCharValue("False") | someCharValue("FALSE") | someCharValue("F"))
 
         escape_space = skip(escape) + space >> joinval
         escape_comma = skip(escape) + comma >> joinval
@@ -175,7 +196,8 @@ class Write(object):
         plain_float_text = someToken('Float') >> tokval
         plain_float = plain_float_text >> (lambda v: float(v))
 
-        identifier = many(char | equal | escape_space | escape_comma |
+        identifier = many(char | plain_float_text | plain_int_text |
+                          escape_space | escape_comma | escape_equal |
                           escape_escape | plain_int_text | quote) >> joinval
         quoted_text_ = many(escape_quote | space | plain_int_text |
                             plain_float_text | char | comma |
@@ -184,14 +206,14 @@ class Write(object):
         unquoted_text = many(escape_space | escape_comma |
                              escape_equal | escape_escape |
                              plain_int_text | char | quote) >> joinval
-        string_value = quoted_text | unquoted_text >> \
-            (lambda s: s in true_values and True
-             or s in false_values and False
-             or s not in false_values and s)
+        boolean_value = (true_value >> (lambda s: True)
+                         | false_value >> (lambda s: False))
+
+        kv_value = plain_int | plain_float | quoted_text | boolean_value
+        kv = (quoted_text | unquoted_text) + skip(equal) + kv_value >> \
+            (lambda x: (x[0], x[1]))
 
-        kv_value = plain_int | plain_float | string_value
-        kv = string_value + \
-            skip(equal) + kv_value >> (lambda x: (x[0], x[1]))
+        tag = identifier + skip(equal) + identifier >> (lambda x: (x[0], x[1]))
 
         def setter(obj, propert):
             def r(val):
@@ -199,7 +221,7 @@ class Write(object):
                 return (propert, val)
             return r
 
-        tags = many(skip(comma) + kv) >> (lambda x: x)
+        tags = many(skip(comma) + tag) >> (lambda x: x)
         fields = (kv + many(skip(comma) + kv)) >> \
                  (lambda x: [x[0]] + x[1])
 
@@ -211,41 +233,53 @@ class Write(object):
             skip(finished) >> (lambda x: x)
 
         result = toplevel.parse(Write.tokenize(line))
-        #pprint(result)
+        # pprint(result)
         return write
 
     def __repr__(self):
         return "<{} key={} tags={} fields={} timestamp={}>".format(
             self.__class__.__name__, self.key, self.tags, self.fields, self.timestamp)
 
-    def __str__(self):
-        def escape_identifier(string):
-            return re.sub(r'([\\, ])', '\\\\\\1', string)
+    @staticmethod
+    def escape_identifier(string):
+        return re.sub(r'([\\,= ])', '\\\\\\1', string)
 
-        def escape_key(string):
-            return re.sub(r'([\\,= ])', '\\\\\\1', string)
+    @staticmethod
+    def escape_tags(taglist):
+        return ",".join(map(lambda kv:
+                            (Write.escape_identifier(kv[0])
+                             + "=" + Write.escape_identifier(kv[1])),
+                            taglist))
 
-        def escape_value(obj):
-            if isinstance(obj, float) or isinstance(obj, int) or isinstance(obj, bool):
-                return str(obj)
-            else:
-                obj = str(obj)
-                return "\"" + obj.replace("\"", "\\\"") + "\""
+    @staticmethod
+    def escape_value(obj):
+        if (isinstance(obj, float)
+                or isinstance(obj, int)
+                or isinstance(obj, bool)):
+            return str(obj)
+        else:
+            obj = str(obj)
+            return "\"" + obj.replace("\\", "\\\\").replace("\"", "\\\"") + "\""
 
-        def escape_kv(kvlist):
-            return ",".join(
-                map(lambda kv: escape_key(kv[0]) + "=" + escape_value(kv[1]),
-                    kvlist))
+    @staticmethod
+    def escape_fields(kvlist):
+        def escape_key(string):
+            return re.sub(r'(["\\,= ])', '\\\\\\1', string)
 
-        result = escape_identifier(self.key)
+        return ",".join(
+            map(lambda kv: escape_key(kv[0]) + "=" + Write.escape_value(kv[1]),
+                kvlist))
+
+    def __str__(self):
+        result = self.escape_identifier(self.key)
 
         if self.tags:
             result += ","
-            result += escape_kv(self.tags)
+            result += self.escape_tags(self.tags)
 
         if self.fields:
             result += " "
-            result += escape_kv(self.fields)
+            result += self.escape_fields(self.fields)
 
         if self.timestamp:
             result += " "