Source Code for Module kenozooid.driver.hwos.parser

  1  # 
  2  # Kenozooid - dive planning and analysis toolbox. 
  3  # 
  4  # Copyright (C) 2009-2019 by Artur Wroblewski <wrobell@riseup.net> 
  5  # 
  6  # This program is free software: you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation, either version 3 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 14  # GNU General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 18  # 
 19   
 20  """ 
 21  Data parser for hwOS family OSTC dive computers driver. 
 22  """ 
 23   
 24  import enum 
 25  import logging 
 26  import operator 
 27  import struct 
 28  from datetime import datetime 
 29  from functools import partial 
 30  from itertools import cycle, takewhile 
 31  from collections import namedtuple 
 32  from cytoolz import itertoolz as itz 
 33  from cytoolz.functoolz import identity 
 34   
 35  import kenozooid.data as kd 
 36  import kenozooid.units as ku 
 37  from kenozooid.util import cumsum 
 38   
 39  logger = logging.getLogger(__name__) 
 40   
 41  COMMANDS = ( 
 42      (b'\x69', 64),     # version identity 
 43      (b'\x60', 5),      # hardware + features 
 44      (b'\x61', 65536),  # dive headers 
 45  ) 
 46   
 47  # page 9 of hwOS interface documentation 
 48  MODEL = { 
 49      0x0a: 'OSTC 3', 
 50      0x1a: 'OSTC 3+', 
 51      0x05: 'OSTC cR', 
 52      0x07: 'OSTC cR', 
 53      0x12: 'OSTC Sport', 
 54      0x11: 'OSTC 2', 
 55      0x13: 'OSTC 2', 
 56      0x1b: 'OSTC 2', 
 57      # 0x13: 'OSTC 3+', 
 58      # 0x13: 'OSTC Sport', 
 59  } 
 60   
 61  # note: gauge -> opencircuit 
 62  DIVE_MODES = 'opencircuit', 'closedcircuit', 'opencircuit', 'apnoe' 
 63   
 64  # the minimal size of a dive profile sample 
 65  # 
 66  # - two bytes for depth 
 67  # - profile flag byte 
 68  MIN_SAMPLE_SIZE = 3 
 69   
 70  # size of each dive profile event 
 71  EVENT_DATA_SIZE = 2, 1, 1, 2 
 72   
 73  # value for disabled gas 
 74  GAS_TYPE_DISABLED = 0 
 75   
 76  # value of first gas mix type as stored in dive header 
 77  GAS_TYPE_FIRST = 1 
 78   
 79  # simple gas mix binary data unpacker: o2 and he values 
 80  UNPACK_GAS_MIX = struct.Struct('<BB').unpack 
 81   
 82  # decompression data binary data unpacker: deco depth and deco time 
 83  UNPACK_DECO = struct.Struct('<BB').unpack 
 84   
 85  RawData = namedtuple('Data', ['version', 'features', 'headers', 'profiles']) 
 86  RawData.__doc__ = """ 
 87  Raw data fetched from a hwOS family OSTC dive computer. 
 88  """ 
 89   
 90  Header = namedtuple( 
 91      'Header', 
 92      [ 
 93          'size', 'datetime', 'depth', 'duration', 'temp', 'gas_list', 
 94          'avg_depth', 'mode' 
 95      ], 
 96  ) 
 97   
 98  ProfileHeader = namedtuple('ProfileHeader', ['size', 'rate', 'divs']) 
 99  Divisor = namedtuple('Divisor', ['type', 'size', 'divisor']) 
100  Events = namedtuple('Events', ['alarm', 'events']) 
101  EventData = namedtuple( 
102      'EventData', 
103      ['manual_gas', 'gas', 'setpoint', 'bailout'] 
104  ) 
105   
106  to_int = partial(int.from_bytes, byteorder='little', signed=False) 
107  to_timestamp = lambda v: datetime(v[0] + 2000, *v[1:]) 
108  to_depth_adj = lambda v: v * 9.80665 / 1000  # TODO: use salinity 
109  to_depth = lambda v: v / 100 
110  to_duration = lambda v: to_int(v[:2]) * 60 + int(v[2]) 
111  to_temp = lambda v: ku.C2K(v / 10) 
112  to_gas = lambda v: kd.gas(v[0], v[1]) 
113  # note: the gas type is needed to determine first gas of a dive 
114  to_gas_list = lambda data: [ 
115      (kd.gas(o2, he, depth), type) 
116      for o2, he, depth, type in itz.partition(4, data) 
117  ] 
118   
119 -class EventFlags(enum.IntFlag): 
120      """ 
121      Event flags. 
122   
123      Event flags do not contain alarm information. 
124   
125      As described at page 6 of hwOS interface documentation. 
126      """ 
127      MANUAL_GAS = 1 
128      GAS = 2 
129      SET_POINT = 4 
130      # 8: always unset, skipping bit indicating next event byte 
131      BAILOUT = 16 
132   
133 -def raw_data(data): 
134      cmd_len = [n for _, n in COMMANDS] 
135      items = partition(data, *cumsum(cmd_len, 0)) 
136      result = RawData(*items) 
137      assert all(len(item) == n for n, item in zip(cmd_len, result)) 
138      return result 
139   
140 -def parse_dives(data): 
141      data = raw_data(data.data) 
142      headers = dive_headers_data(data.headers) 
143      headers = [parse_header(v) for v in headers] 
144   
145      # note: hwos header size stated in header raw data includes 3 more 
146      # bytes, why? 
147      idx = cumsum((h.size - 3 for h in headers), 0) 
148      profiles = partition(data.profiles, *idx) 
149      yield from (create_dive(h, p) for h, p in zip(headers, profiles)) 
150   
151 -def parse_profile(header, data): 
152      """ 
153      Parse dive profile raw data. 
154   
155      :param header: Dive header. 
156      :param data: Dive profile raw data. 
157      """ 
158      assert data[-2:] == b'\xfd\xfd', data[-2:] 
159   
160      p_header = parse_profile_header(data) 
161      assert p_header.size == header.size 
162      ext_parsers = create_extended_data_parsers(p_header) 
163   
164      start = len(p_header.divs) * 3 + 5 
165      profile = data[start:-2] 
166   
167      idx = dive_profile_sample_idx(profile) 
168      samples = partition(profile, *idx) 
169      samples = zip(enumerate(samples, 1), ext_parsers) 
170      samples = ((k, s, p) for (k, s), p in samples)  # flatten above zip 
171      samples = (create_sample(header, p_header, p, k, s) for k, s, p in samples) 
172      # omit shallow/surface part 
173      samples = takewhile(lambda s: s.time <= header.duration, samples) 
174   
175      first_gas = get_first_gas(header) 
176   
177      yield kd.Sample(depth=0, time=0, gas=first_gas) 
178      yield from samples 
179      yield kd.Sample(depth=0, time=header.duration + p_header.rate) 
180   
181 -def get_first_gas(header): 
182      """ 
183      Get first gas from dive header. 
184   
185      If there is no first gas configured, then pick first non-disabled gas. 
186      """ 
187      assert header.gas_list 
188      gas_mixes = (m for m, t in header.gas_list if t == GAS_TYPE_FIRST) 
189      first_gas = next(gas_mixes, None) 
190      if first_gas is None: 
191          logger.warning( 
192              'Dive {}, no first gas, picking first non-disabled' 
193              .format(header.datetime) 
194          ) 
195          gas_mixes = (m for m, t in header.gas_list if t != GAS_TYPE_DISABLED) 
196          first_gas = next(gas_mixes, None) 
197      assert first_gas is not None 
198      return first_gas 
199   
200 -def parse_events(data): 
201      """ 
202      Parse event data from event bytes of dive profile sample raw data. 
203   
204      :param data: Dive profile sample raw data. 
205      """ 
206      items = takewhile(lambda v: v & 0x80, data[2:]) 
207      k = itz.count(items) 
208   
209      event_bytes_start = 3 + k 
210      items = data[3:event_bytes_start] 
211      value = to_int(v & ~0x80 for v in items) 
212   
213      alarm = value & 0x07 
214      events = EventFlags(value >> 4) 
215   
216      event_data = parse_event_data(events, data[event_bytes_start:]) 
217      return Events(alarm, event_data) 
218   
219 -def parse_event_data(events, data): 
220      """ 
221      Parse event data from a raw data of a sample of a dive profile. 
222   
223      :param events: Event flags. 
224      :param data: Raw data of a sample of a dive profile. 
225      """ 
226      items = zip(EventFlags, EVENT_DATA_SIZE) 
227      # keep event byte size if flag is set, zero it if not, so non-existing 
228      # event byte results in empty string 
229      items = (bool(events & f) * s for f, s in items) 
230      idx = cumsum(items, 0) 
231   
232      manual_gas, gas, setpoint, bailout, *_ = partition(data, *idx) 
233      manual_gas = to_gas(UNPACK_GAS_MIX(manual_gas)) if manual_gas else None 
234      gas = to_int(gas) if gas else None 
235      setpoint = to_int(setpoint) / 100 if setpoint else None 
236      bailout = to_gas(UNPACK_GAS_MIX(bailout)) if bailout else None 
237      return EventData(manual_gas, gas, setpoint, bailout) 
238   
239 -def create_extended_data_parsers(profile_header): 
240      """ 
241      Create iterator of parsers for dive profile extended data. 
242   
243      Dive profile samples contains variable amount of extended data and 
244      require a specific parser. 
245   
246      :param profile_header: Dive profile header. 
247      """ 
248      # determine divisors for which data will exist 
249      divs = set(d.divisor for d in profile_header.divs) 
250      max_div = max(divs) if divs else 0 
251   
252      # create parser for each sample 1..max_div 
253      parsers = ( 
254          create_extended_data_parser(profile_header, i) 
255          for i in range(1, max_div + 1) 
256      ) 
257      # cycle all parsers 
258      return cycle(parsers) 
259   
260 -def create_extended_data_parser(profile_header, sample_no): 
261      """ 
262      Create extended data parser for a specific dive profile sample. 
263   
264      Divisor information is used to determine how data should be parsed for 
265      given dive profile sample number. 
266   
267      :param profile_header: Dive profile header. 
268      :param sample_no: Dive profile sample number. 
269      """ 
270      sizes = ( 
271          d.size if d.divisor and sample_no % d.divisor == 0 else 0 
272          for d in profile_header.divs 
273      ) 
274      idx = list(cumsum(sizes, 0)) 
275   
276      div_type_parser = { 
277          0: lambda v: to_temp(struct.unpack('<h', v)[0]), 
278          1: UNPACK_DECO, 
279          2: lambda v: None, 
280          3: lambda v: None, 
281          4: lambda v: None, 
282          5: lambda v: None, 
283          6: lambda v: None, 
284      } 
285      # determine parser for each divisor type 
286      parsers = [div_type_parser[d.type] for d in profile_header.divs] 
287   
288      def parser(data): 
289          # split data into extended information items 
290          items = partition(data, *idx) 
291          # parse each extended information item 
292          return tuple(f(v) if v else None for f, v in zip(parsers, items)) 
293   
294      return parser 
295   
296 -def model_version(data): 
297      """ 
298      Get model and version information about a hwOS family OSTC dive 
299      computer. 
300   
301      :param data: Raw data fetched from a hwOS family OSTC dive computer. 
302   
303      .. seealso:: `RawData` 
304      """ 
305      major, minor = data.version[2:4] 
306      dsc = data.features[1] 
307      model = MODEL.get(dsc, 'OSTC hwOS') 
308      logger.debug('descriptor 0x{:x} -> model {}'.format(dsc, model)) 
309      return model, major, minor 
310   
311 -def parse_header(data): 
312      """ 
313      Parse dive header data read from hwOS OSTC dive computer. 
314      """ 
315      parsers = ( 
316          identity,  # start marker 
317          to_int, to_timestamp, to_depth, to_duration, to_temp, 
318          to_gas_list, 
319          # average depth, dive mode 
320          to_depth, identity, 
321          identity  # end marker 
322      ) 
323      # 5s - datetime, 20s - gas list, B - dive mode 
324      fmt = '<H 7x 3s 5s H 3s h 4x 20s 25x H 7x B 171x H' 
325      fields = header_fields(parsers, fmt, data) 
326      return Header._make(fields) 
327   
328 -def parse_profile_header(data): 
329      """ 
330      Parse dive profile header (aka small header). 
331   
332      :param data: Dive profile data. 
333      """ 
334      size, rate, no_div = struct.unpack('3sBB', data[:5]) 
335      size = to_int(size) 
336   
337      divs = data[5:5 + 3 * no_div] 
338      assert len(divs) == 3 * no_div 
339      divs = struct.unpack('BBB' * no_div, divs) 
340      divs = itz.partition(3, divs) 
341      divs = tuple(Divisor._make(v) for v in divs) 
342   
343      return ProfileHeader(size, rate, divs) 
344   
345 -def dive_profile_size(data): 
346      """ 
347      Extract dive profile size from dive header raw data. 
348      """ 
349      parsers = (identity, to_int, identity) 
350      fmt = '<H7x3s242xH' 
351      data = header_fields(parsers, fmt, data) 
352      return data[0] 
353   
354 -def dive_headers_data(headers): 
355      """ 
356      Divide all dive header raw data into collection of raw data for each 
357      header. 
358   
359      :param headers: Raw header data fetched from hwOS OSTC dive computer. 
360      """ 
361      assert len(headers) == 65536 
362   
363      items = itz.partition(256, headers) 
364      # convert back to bytes, see 
365      # 
366      # - https://github.com/pytoolz/cytoolz/issues/102 
367      # - https://github.com/pytoolz/toolz/issues/377 
368      # 
369      # also filter unused headers via `logbook-profile version` header field 
370      yield from (bytes(v) for v in items if v[8] != 0xff) 
371   
372 -def header_fields(parsers, fmt, data): 
373      """ 
374      Extract dive header fields data from dive header raw data. 
375   
376      :param parsers: Parser for each field, including start and end markers. 
377      :param fmt: Struct format to extract the fields. 
378      :param data: Dive header raw data. 
379      """ 
380      assert struct.calcsize(fmt) == 256, struct.calcsize(fmt) 
381   
382      items = struct.unpack(fmt, data) 
383      items = zip(parsers, items) 
384      start, *fields, end = (p(v) for p, v in items) 
385      assert start == 0xfafa and end == 0xfbfb, '0x{:x} 0x{:x}'.format(start, end) 
386   
387      return fields 
388   
389 -def dive_profile_sample_idx(data): 
390      """ 
391      Determine index of each dive profile sample using the profile raw data. 
392   
393      :param data: Dive profile raw data. 
394      """ 
395      # there is no more than `n_samples` dive profile samples 
396      n_samples = len(data) // MIN_SAMPLE_SIZE 
397   
398      f = partial(dive_profile_next_sample, data) 
399      idx = itz.accumulate(f, range(n_samples)) 
400      idx = takewhile(lambda i: i < len(data), idx) 
401      return idx 
402   
403 -def dive_profile_next_sample(data, idx, sample_no): 
404      """ 
405      Calculate index of next dive profile sample. 
406   
407      The calculation is performed using dive profile raw data and index of 
408      current dive profile sample. The `idx + 2` points to profile flag byte, 
409      which is used to determine total length of current sample. 
410   
411      :param data: Dive profile raw data. 
412      :param idx: Index of current dive profile sample. 
413      :param sample_no: Sample number (unused). 
414      """ 
415      return idx + (data[idx + 2] & 0x7f) + MIN_SAMPLE_SIZE 
416   
417 -def partition(data, *idx): 
418      """ 
419      Partition the data using indexes. 
420   
421      Each index is start of each item. 
422   
423      :param data: Data to partition. 
424      :param idx: Indexes used to partition the data. 
425      """ 
426      item_range = idx + (None,) 
427      item_range = zip(item_range[:-1], item_range[1:]) 
428      yield from (data[j:k] for j, k in item_range) 
429   
430 -def create_dive(header, data): 
431      """ 
432      Create Kenozooid dive record. 
433   
434      :param header: Dive header of a dive stored in hwOS OSTC dive computer. 
435      :param data: Dive profile raw data. 
436      """ 
437      return kd.Dive( 
438          datetime=header.datetime, 
439          depth=header.depth, 
440          duration=header.duration, 
441          temp=header.temp, 
442          avg_depth=header.avg_depth, 
443          mode=DIVE_MODES[header.mode], 
444          profile=list(parse_profile(header, data)), 
445      ) 
446   
447 -def create_sample(header, profile_header, ext_parser, sample_no, data): 
448      """ 
449      Create Kenozooid dive profile sample record. 
450   
451      :param header: Dive header. 
452      :param profile_header: Dive profile header. 
453      :param ext_parser: Extended information parser. 
454      :param sample_no: Dive profile sample number. 
455      :param data: Dive profile sample data. 
456      """ 
457      depth = to_depth_adj(to_int(data[:2])) 
458      time = sample_no * profile_header.rate 
459   
460      events = parse_events(data) 
461   
462      gas_no = events.events.gas 
463      assert gas_no is None or gas_no > 0 
464      gas = header.gas_list[gas_no - 1][0] if events.events.gas else None 
465      # probably not possible, but overwrite current gas mix with the mix set 
466      # manually 
467      gas = events.events.manual_gas if events.events.manual_gas else gas 
468   
469      start = sum(bool(v) * s for v, s in zip(events.events, EVENT_DATA_SIZE)) 
470      start += MIN_SAMPLE_SIZE  # include depth and profile byte 
471   
472      # parse extended information 
473      temp, deco, *_ = ext_parser(data[start:]) 
474      deco_depth, deco_time = deco if deco and deco[0] else (None, None) 
475   
476      if __debug__ and gas: 
477          logger.debug('Dive {}, gas number: {}, gas: {}'.format( 
478              header.datetime, events.events.gas, gas 
479          )) 
480   
481      alarm = ('deco',) if events.alarm == 2 else None 
482      return kd.Sample( 
483          depth=depth, 
484          time=time, 
485          temp=temp, 
486          deco_time=deco_time, 
487          deco_depth=deco_depth, 
488          alarm=alarm, 
489          gas=gas, 
490      ) 
491   
492  # vim: sw=4:et:ai 
493