From bcd865c7c893f28ff4bfac2c92eda4486326d4aa Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Sun, 18 Jun 2017 13:00:43 +1000 Subject: [PATCH 01/12] Update InfluxDB to handle datetime objects Updates the InfluxDB regex to ignore datetime objects being coverted into float values. Adds tests to the component to ensure datetime objects are corectly handled. --- homeassistant/components/influxdb.py | 5 ++++- tests/components/test_influxdb.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 37aeeee41b7bb7..4a3a135ec95591 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -8,6 +8,8 @@ import re +import datetime + import voluptuous as vol from homeassistant.const import ( @@ -165,7 +167,8 @@ def influx_event_listener(event): except (ValueError, TypeError): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) - if non_digit_tail.match(json_body[0]['fields'][new_key]): + if non_digit_tail.match(json_body[0]['fields'][new_key]) and not isinstance( + value, datetime.datetime): json_body[0]['fields'][key] = float( non_decimal.sub('', value)) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index 301c4c7b9b1ecf..45066985e6ccab 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -1,5 +1,6 @@ """The tests for the InfluxDB component.""" import unittest +import datetime from unittest import mock import influxdb as influx_client @@ -123,7 +124,8 @@ def test_event_listener(self, mock_client): 'latitude': '2.2', 'battery_level': '99%', 'temperature': '20c', - 'last_seen': 'Last seen 23 minutes ago' + 'last_seen': 'Last seen 23 minutes ago', + 'updated_at': datetime.datetime(2017, 1, 1, 0, 0) } state = mock.MagicMock( state=in_, domain='fake', object_id='entity', attributes=attrs) @@ -144,7 +146,8 @@ def test_event_listener(self, mock_client): 'battery_level': 99.0, 'temperature_str': '20c', 'temperature': 20.0, - 'last_seen_str': 'Last seen 23 minutes ago' + 'last_seen_str': 'Last seen 23 minutes ago', + 'updated_at_str': '2017-01-01 00:00:00' }, }] @@ -164,7 +167,8 @@ def test_event_listener(self, mock_client): 'battery_level': 99.0, 'temperature_str': '20c', 'temperature': 20.0, - 'last_seen_str': 'Last seen 23 minutes ago' + 'last_seen_str': 'Last seen 23 minutes ago', + 'updated_at_str': '2017-01-01 00:00:00' }, }] self.handler_method(event) From f599c393945b40a7332adb34d6d724dc6ff02405 Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Sun, 18 Jun 2017 13:23:58 +1000 Subject: [PATCH 02/12] Fix Hound errors Fixes errors from Hound bot --- homeassistant/components/influxdb.py | 5 +++-- tests/components/test_influxdb.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 4a3a135ec95591..30f1af0bb8520c 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -167,8 +167,9 @@ def influx_event_listener(event): except (ValueError, TypeError): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) - if non_digit_tail.match(json_body[0]['fields'][new_key]) and not isinstance( - value, datetime.datetime): + if non_digit_tail.match( + json_body[0]['fields'][new_key]) and not isinstance( + value, datetime.datetime): json_body[0]['fields'][key] = float( non_decimal.sub('', value)) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index 45066985e6ccab..1d246d48eab664 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -168,7 +168,7 @@ def test_event_listener(self, mock_client): 'temperature_str': '20c', 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', - 'updated_at_str': '2017-01-01 00:00:00' + 'updated_at_str': '2017-01-01 00:00:00' }, }] self.handler_method(event) From 057f150d23818c3969d7c58ebfe4d42bfded06a3 Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Sun, 18 Jun 2017 14:48:11 +1000 Subject: [PATCH 03/12] Update InfluxDB to handle multiple decimal points Changes the way InfluxDB handles values such as 1.2.3.4 to be 1.234 so it stores in InfluxDB as a valid float value --- homeassistant/components/influxdb.py | 12 ++++++++++-- tests/components/test_influxdb.py | 11 ++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 30f1af0bb8520c..14e51814fce880 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -107,6 +107,10 @@ def setup(hass, config): "the database exists and is READ/WRITE.", exc) return False + def decreplace(matchobj): + """Convert values such as 1.2.3.4 to 1.234 for Influx""" + return matchobj.group(1) + re.sub(r'\.', '', matchobj.group(2)) + def influx_event_listener(event): """Listen for new messages on the bus and sends them to Influx.""" state = event.data.get('new_state') @@ -170,8 +174,12 @@ def influx_event_listener(event): if non_digit_tail.match( json_body[0]['fields'][new_key]) and not isinstance( value, datetime.datetime): - json_body[0]['fields'][key] = float( - non_decimal.sub('', value)) + try: + json_body[0]['fields'][key] = float( + non_decimal.sub('', value)) + except (ValueError, TypeError): + json_body[0]['fields'][key] = float( + re.sub(r'^([^.]*\.)(.*)$', decreplace, value)) json_body[0]['tags'].update(tags) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index 1d246d48eab664..c4594a800a4e9a 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -125,7 +125,8 @@ def test_event_listener(self, mock_client): 'battery_level': '99%', 'temperature': '20c', 'last_seen': 'Last seen 23 minutes ago', - 'updated_at': datetime.datetime(2017, 1, 1, 0, 0) + 'updated_at': datetime.datetime(2017, 1, 1, 0, 0), + 'multi_periods': '0.120.240.2023873' } state = mock.MagicMock( state=in_, domain='fake', object_id='entity', attributes=attrs) @@ -147,7 +148,9 @@ def test_event_listener(self, mock_client): 'temperature_str': '20c', 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', - 'updated_at_str': '2017-01-01 00:00:00' + 'updated_at_str': '2017-01-01 00:00:00', + 'multi_periods_str': '0.120.240.2023873', + 'multi_periods': 0.1202402023873 }, }] @@ -168,7 +171,9 @@ def test_event_listener(self, mock_client): 'temperature_str': '20c', 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', - 'updated_at_str': '2017-01-01 00:00:00' + 'updated_at_str': '2017-01-01 00:00:00', + 'multi_periods_str': '0.120.240.2023873', + 'multi_periods': 0.1202402023873 }, }] self.handler_method(event) From 1c1ebddc64085d590a2db4f6cc3f233c71a3b80b Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Sun, 18 Jun 2017 14:51:21 +1000 Subject: [PATCH 04/12] Fix lint issues Reduce the size of a line for the linter --- homeassistant/components/influxdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 14e51814fce880..787fbcad092044 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -172,8 +172,8 @@ def influx_event_listener(event): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) if non_digit_tail.match( - json_body[0]['fields'][new_key]) and not isinstance( - value, datetime.datetime): + json_body[0]['fields'][new_key]) and not ( + isinstance(value, datetime.datetime)): try: json_body[0]['fields'][key] = float( non_decimal.sub('', value)) From 74ea240dc05d2d3e1fe6161146f7d0e168946a19 Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Sun, 18 Jun 2017 20:42:50 +1000 Subject: [PATCH 05/12] Update InfluxDB to pass on unknown variable If we get an error trying to convert a variable to a float, let's ignore it completely --- homeassistant/components/influxdb.py | 12 ++---------- tests/components/test_influxdb.py | 6 ++---- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 787fbcad092044..0cb215f66ce288 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -8,8 +8,6 @@ import re -import datetime - import voluptuous as vol from homeassistant.const import ( @@ -107,10 +105,6 @@ def setup(hass, config): "the database exists and is READ/WRITE.", exc) return False - def decreplace(matchobj): - """Convert values such as 1.2.3.4 to 1.234 for Influx""" - return matchobj.group(1) + re.sub(r'\.', '', matchobj.group(2)) - def influx_event_listener(event): """Listen for new messages on the bus and sends them to Influx.""" state = event.data.get('new_state') @@ -172,14 +166,12 @@ def influx_event_listener(event): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) if non_digit_tail.match( - json_body[0]['fields'][new_key]) and not ( - isinstance(value, datetime.datetime)): + json_body[0]['fields'][new_key]): try: json_body[0]['fields'][key] = float( non_decimal.sub('', value)) except (ValueError, TypeError): - json_body[0]['fields'][key] = float( - re.sub(r'^([^.]*\.)(.*)$', decreplace, value)) + pass json_body[0]['tags'].update(tags) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index c4594a800a4e9a..770b457beca648 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -149,8 +149,7 @@ def test_event_listener(self, mock_client): 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', 'updated_at_str': '2017-01-01 00:00:00', - 'multi_periods_str': '0.120.240.2023873', - 'multi_periods': 0.1202402023873 + 'multi_periods_str': '0.120.240.2023873' }, }] @@ -172,8 +171,7 @@ def test_event_listener(self, mock_client): 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', 'updated_at_str': '2017-01-01 00:00:00', - 'multi_periods_str': '0.120.240.2023873', - 'multi_periods': 0.1202402023873 + 'multi_periods_str': '0.120.240.2023873' }, }] self.handler_method(event) From 271479899f07dfa3e749c043396e73cb1311335b Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Mon, 19 Jun 2017 09:31:23 +1000 Subject: [PATCH 06/12] Make InfluxDB Regex constants Makes the Regex's used by InfluxDB constants so they don't need to be compiled each time --- homeassistant/components/influxdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 0cb215f66ce288..77b398cdd7b2df 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -58,6 +58,8 @@ }), }, extra=vol.ALLOW_EXTRA) +NON_DIGIT_TAIL = re.compile(r'[\d.]+') +NON_DECIMAL = re.compile(r'[^\d.]+') def setup(hass, config): """Set up the InfluxDB component.""" @@ -149,8 +151,6 @@ def influx_event_listener(event): } ] - non_digit_tail = re.compile(r'[\d.]+') - non_decimal = re.compile(r'[^\d.]+') for key, value in state.attributes.items(): if key != 'unit_of_measurement': # If the key is already in fields @@ -165,11 +165,11 @@ def influx_event_listener(event): except (ValueError, TypeError): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) - if non_digit_tail.match( + if NON_DIGIT_TAIL.match( json_body[0]['fields'][new_key]): try: json_body[0]['fields'][key] = float( - non_decimal.sub('', value)) + NON_DECIMAL.sub('', value)) except (ValueError, TypeError): pass From 5e5961cc1506780b7ed8d25518b6e454f53936d4 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 19 Jun 2017 09:48:43 +0200 Subject: [PATCH 07/12] cleanup --- homeassistant/components/influxdb.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 77b398cdd7b2df..ab4cb34353a969 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -58,8 +58,8 @@ }), }, extra=vol.ALLOW_EXTRA) -NON_DIGIT_TAIL = re.compile(r'[\d.]+') -NON_DECIMAL = re.compile(r'[^\d.]+') +RE_DIGIT_TAIL = re.compile(r'^\w*\d+\.?\d+\w*$') +RE_DECIMAL = re.compile(r'[^\d.]+') def setup(hass, config): """Set up the InfluxDB component.""" @@ -165,13 +165,10 @@ def influx_event_listener(event): except (ValueError, TypeError): new_key = "{}_str".format(key) json_body[0]['fields'][new_key] = str(value) - if NON_DIGIT_TAIL.match( - json_body[0]['fields'][new_key]): - try: - json_body[0]['fields'][key] = float( - NON_DECIMAL.sub('', value)) - except (ValueError, TypeError): - pass + + if RE_DIGIT_TAIL.match(json_body[0]['fields'][new_key]): + json_body[0]['fields'][key] = float( + RE_DECIMAL.sub('', value)) json_body[0]['tags'].update(tags) From 5dfd66189b0c5bb8c21a158cfa43e73e4a0887cd Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 19 Jun 2017 23:56:38 +0200 Subject: [PATCH 08/12] fix lint --- homeassistant/components/influxdb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index ab4cb34353a969..cdbabf7c7c4e8b 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -61,6 +61,7 @@ RE_DIGIT_TAIL = re.compile(r'^\w*\d+\.?\d+\w*$') RE_DECIMAL = re.compile(r'[^\d.]+') + def setup(hass, config): """Set up the InfluxDB component.""" from influxdb import InfluxDBClient, exceptions From 974774e8de5fb4d50331f940b398e2ee68a6349f Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 20 Jun 2017 00:04:24 +0200 Subject: [PATCH 09/12] Update regex --- homeassistant/components/influxdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index cdbabf7c7c4e8b..63dc090aa4576d 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -58,7 +58,7 @@ }), }, extra=vol.ALLOW_EXTRA) -RE_DIGIT_TAIL = re.compile(r'^\w*\d+\.?\d+\w*$') +RE_DIGIT_TAIL = re.compile(r'^[^\.]*\d+\.?\d+[^\.]*$') RE_DECIMAL = re.compile(r'[^\d.]+') From 803eda750371849979f14f3d285146226cedcc40 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 20 Jun 2017 00:10:20 +0200 Subject: [PATCH 10/12] fix tests --- tests/components/test_influxdb.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index 770b457beca648..a2d31ff7fc3497 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -148,7 +148,9 @@ def test_event_listener(self, mock_client): 'temperature_str': '20c', 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', + 'last_seen': 23.0, 'updated_at_str': '2017-01-01 00:00:00', + 'updated_at': 20170101000000, 'multi_periods_str': '0.120.240.2023873' }, }] @@ -170,9 +172,10 @@ def test_event_listener(self, mock_client): 'temperature_str': '20c', 'temperature': 20.0, 'last_seen_str': 'Last seen 23 minutes ago', + 'last_seen': 23.0, 'updated_at_str': '2017-01-01 00:00:00', - 'multi_periods_str': '0.120.240.2023873' - }, + 'updated_at': 20170101000000, + 'multi_periods_str': '0.120.240.2023873' }, }] self.handler_method(event) self.assertEqual( From b585daf15cadc0a42ab2685e633754697e076d47 Mon Sep 17 00:00:00 2001 From: Phil Hawthorne Date: Tue, 20 Jun 2017 09:29:44 +1000 Subject: [PATCH 11/12] Fix JSON body missing new line character --- tests/components/test_influxdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/components/test_influxdb.py b/tests/components/test_influxdb.py index a2d31ff7fc3497..896ee4699ccb00 100644 --- a/tests/components/test_influxdb.py +++ b/tests/components/test_influxdb.py @@ -175,7 +175,8 @@ def test_event_listener(self, mock_client): 'last_seen': 23.0, 'updated_at_str': '2017-01-01 00:00:00', 'updated_at': 20170101000000, - 'multi_periods_str': '0.120.240.2023873' }, + 'multi_periods_str': '0.120.240.2023873' + }, }] self.handler_method(event) self.assertEqual( From d523faddfae24c9b45eefc9978540c17b70689bb Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 20 Jun 2017 07:33:33 +0200 Subject: [PATCH 12/12] fix exceptions --- homeassistant/components/influxdb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/influxdb.py b/homeassistant/components/influxdb.py index 63dc090aa4576d..6b8cd6734dd34b 100644 --- a/homeassistant/components/influxdb.py +++ b/homeassistant/components/influxdb.py @@ -165,11 +165,12 @@ def influx_event_listener(event): json_body[0]['fields'][key] = float(value) except (ValueError, TypeError): new_key = "{}_str".format(key) - json_body[0]['fields'][new_key] = str(value) + new_value = str(value) + json_body[0]['fields'][new_key] = new_value - if RE_DIGIT_TAIL.match(json_body[0]['fields'][new_key]): + if RE_DIGIT_TAIL.match(new_value): json_body[0]['fields'][key] = float( - RE_DECIMAL.sub('', value)) + RE_DECIMAL.sub('', new_value)) json_body[0]['tags'].update(tags)