Package kenozooid :: Package driver :: Package hwos :: Module parser

Source Code for Module kenozooid.driver.hwos.parser

  1  # 
  2  # Kenozooid - dive planning and analysis toolbox. 
  3  # 
  4  # Copyright (C) 2009-2019 by Artur Wroblewski <wrobell@riseup.net> 
  5  # 
  6  # This program is free software: you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation, either version 3 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 14  # GNU General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 18  # 
 19   
 20  """ 
 21  Data parser for hwOS family OSTC dive computers driver. 
 22  """ 
 23   
 24  import enum 
 25  import logging 
 26  import operator 
 27  import struct 
 28  from datetime import datetime 
 29  from functools import partial 
 30  from itertools import cycle, takewhile 
 31  from collections import namedtuple 
 32  from cytoolz import itertoolz as itz 
 33  from cytoolz.functoolz import identity 
 34   
 35  import kenozooid.data as kd 
 36  import kenozooid.units as ku 
 37  from kenozooid.util import cumsum 
 38   
 39  logger = logging.getLogger(__name__) 
 40   
 41  COMMANDS = ( 
 42      (b'\x69', 64),     # version identity 
 43      (b'\x60', 5),      # hardware + features 
 44      (b'\x61', 65536),  # dive headers 
 45  ) 
 46   
 47  # page 9 of hwOS interface documentation 
 48  MODEL = { 
 49      0x0a: 'OSTC 3', 
 50      0x1a: 'OSTC 3+', 
 51      0x05: 'OSTC cR', 
 52      0x07: 'OSTC cR', 
 53      0x12: 'OSTC Sport', 
 54      0x11: 'OSTC 2', 
 55      0x13: 'OSTC 2', 
 56      0x1b: 'OSTC 2', 
 57      # 0x13: 'OSTC 3+', 
 58      # 0x13: 'OSTC Sport', 
 59  } 
 60   
 61  # note: gauge -> opencircuit 
 62  DIVE_MODES = 'opencircuit', 'closedcircuit', 'opencircuit', 'apnoe' 
 63   
 64  # the minimal size of a dive profile sample 
 65  # 
 66  # - two bytes for depth 
 67  # - profile flag byte 
 68  MIN_SAMPLE_SIZE = 3 
 69   
 70  # size of each dive profile event 
 71  EVENT_DATA_SIZE = 2, 1, 1, 2 
 72   
 73  # value for disabled gas 
 74  GAS_TYPE_DISABLED = 0 
 75   
 76  # value of first gas mix type as stored in dive header 
 77  GAS_TYPE_FIRST = 1 
 78   
 79  # simple gas mix binary data unpacker: o2 and he values 
 80  UNPACK_GAS_MIX = struct.Struct('<BB').unpack 
 81   
 82  # decompression data binary data unpacker: deco depth and deco time 
 83  UNPACK_DECO = struct.Struct('<BB').unpack 
 84   
 85  RawData = namedtuple('Data', ['version', 'features', 'headers', 'profiles']) 
 86  RawData.__doc__ = """ 
 87  Raw data fetched from a hwOS family OSTC dive computer. 
 88  """ 
 89   
 90  Header = namedtuple( 
 91      'Header', 
 92      [ 
 93          'size', 'datetime', 'depth', 'duration', 'temp', 'gas_list', 
 94          'avg_depth', 'mode' 
 95      ], 
 96  ) 
 97   
 98  ProfileHeader = namedtuple('ProfileHeader', ['size', 'rate', 'divs']) 
 99  Divisor = namedtuple('Divisor', ['type', 'size', 'divisor']) 
100  Events = namedtuple('Events', ['alarm', 'events']) 
101  EventData = namedtuple( 
102      'EventData', 
103      ['manual_gas', 'gas', 'setpoint', 'bailout'] 
104  ) 
105   
106  to_int = partial(int.from_bytes, byteorder='little', signed=False) 
107  to_timestamp = lambda v: datetime(v[0] + 2000, *v[1:]) 
108  to_depth_adj = lambda v: v * 9.80665 / 1000  # TODO: use salinity 
109  to_depth = lambda v: v / 100 
110  to_duration = lambda v: to_int(v[:2]) * 60 + int(v[2]) 
111  to_temp = lambda v: ku.C2K(v / 10) 
112  to_gas = lambda v: kd.gas(v[0], v[1]) 
113  # note: the gas type is needed to determine first gas of a dive 
114  to_gas_list = lambda data: [ 
115      (kd.gas(o2, he, depth), type) 
116      for o2, he, depth, type in itz.partition(4, data) 
117  ] 
118   
119 -class EventFlags(enum.IntFlag):
120 """ 121 Event flags. 122 123 Event flags do not contain alarm information. 124 125 As described at page 6 of hwOS interface documentation. 126 """ 127 MANUAL_GAS = 1 128 GAS = 2 129 SET_POINT = 4 130 # 8: always unset, skipping bit indicating next event byte 131 BAILOUT = 16
132
133 -def raw_data(data):
134 cmd_len = [n for _, n in COMMANDS] 135 items = partition(data, *cumsum(cmd_len, 0)) 136 result = RawData(*items) 137 assert all(len(item) == n for n, item in zip(cmd_len, result)) 138 return result
139
140 -def parse_dives(data):
141 data = raw_data(data.data) 142 headers = dive_headers_data(data.headers) 143 headers = [parse_header(v) for v in headers] 144 145 # note: hwos header size stated in header raw data includes 3 more 146 # bytes, why? 147 idx = cumsum((h.size - 3 for h in headers), 0) 148 profiles = partition(data.profiles, *idx) 149 yield from (create_dive(h, p) for h, p in zip(headers, profiles))
150
151 -def parse_profile(header, data):
152 """ 153 Parse dive profile raw data. 154 155 :param header: Dive header. 156 :param data: Dive profile raw data. 157 """ 158 assert data[-2:] == b'\xfd\xfd', data[-2:] 159 160 p_header = parse_profile_header(data) 161 assert p_header.size == header.size 162 ext_parsers = create_extended_data_parsers(p_header) 163 164 start = len(p_header.divs) * 3 + 5 165 profile = data[start:-2] 166 167 idx = dive_profile_sample_idx(profile) 168 samples = partition(profile, *idx) 169 samples = zip(enumerate(samples, 1), ext_parsers) 170 samples = ((k, s, p) for (k, s), p in samples) # flatten above zip 171 samples = (create_sample(header, p_header, p, k, s) for k, s, p in samples) 172 # omit shallow/surface part 173 samples = takewhile(lambda s: s.time <= header.duration, samples) 174 175 first_gas = get_first_gas(header) 176 177 yield kd.Sample(depth=0, time=0, gas=first_gas) 178 yield from samples 179 yield kd.Sample(depth=0, time=header.duration + p_header.rate)
180
181 -def get_first_gas(header):
182 """ 183 Get first gas from dive header. 184 185 If there is no first gas configured, then pick first non-disabled gas. 186 """ 187 assert header.gas_list 188 gas_mixes = (m for m, t in header.gas_list if t == GAS_TYPE_FIRST) 189 first_gas = next(gas_mixes, None) 190 if first_gas is None: 191 logger.warning( 192 'Dive {}, no first gas, picking first non-disabled' 193 .format(header.datetime) 194 ) 195 gas_mixes = (m for m, t in header.gas_list if t != GAS_TYPE_DISABLED) 196 first_gas = next(gas_mixes, None) 197 assert first_gas is not None 198 return first_gas
199
200 -def parse_events(data):
201 """ 202 Parse event data from event bytes of dive profile sample raw data. 203 204 :param data: Dive profile sample raw data. 205 """ 206 items = takewhile(lambda v: v & 0x80, data[2:]) 207 k = itz.count(items) 208 209 event_bytes_start = 3 + k 210 items = data[3:event_bytes_start] 211 value = to_int(v & ~0x80 for v in items) 212 213 alarm = value & 0x07 214 events = EventFlags(value >> 4) 215 216 event_data = parse_event_data(events, data[event_bytes_start:]) 217 return Events(alarm, event_data)
218
219 -def parse_event_data(events, data):
220 """ 221 Parse event data from a raw data of a sample of a dive profile. 222 223 :param events: Event flags. 224 :param data: Raw data of a sample of a dive profile. 225 """ 226 items = zip(EventFlags, EVENT_DATA_SIZE) 227 # keep event byte size if flag is set, zero it if not, so non-existing 228 # event byte results in empty string 229 items = (bool(events & f) * s for f, s in items) 230 idx = cumsum(items, 0) 231 232 manual_gas, gas, setpoint, bailout, *_ = partition(data, *idx) 233 manual_gas = to_gas(UNPACK_GAS_MIX(manual_gas)) if manual_gas else None 234 gas = to_int(gas) if gas else None 235 setpoint = to_int(setpoint) / 100 if setpoint else None 236 bailout = to_gas(UNPACK_GAS_MIX(bailout)) if bailout else None 237 return EventData(manual_gas, gas, setpoint, bailout)
238
239 -def create_extended_data_parsers(profile_header):
240 """ 241 Create iterator of parsers for dive profile extended data. 242 243 Dive profile samples contains variable amount of extended data and 244 require a specific parser. 245 246 :param profile_header: Dive profile header. 247 """ 248 # determine divisors for which data will exist 249 divs = set(d.divisor for d in profile_header.divs) 250 max_div = max(divs) if divs else 0 251 252 # create parser for each sample 1..max_div 253 parsers = ( 254 create_extended_data_parser(profile_header, i) 255 for i in range(1, max_div + 1) 256 ) 257 # cycle all parsers 258 return cycle(parsers)
259
260 -def create_extended_data_parser(profile_header, sample_no):
261 """ 262 Create extended data parser for a specific dive profile sample. 263 264 Divisor information is used to determine how data should be parsed for 265 given dive profile sample number. 266 267 :param profile_header: Dive profile header. 268 :param sample_no: Dive profile sample number. 269 """ 270 sizes = ( 271 d.size if d.divisor and sample_no % d.divisor == 0 else 0 272 for d in profile_header.divs 273 ) 274 idx = list(cumsum(sizes, 0)) 275 276 div_type_parser = { 277 0: lambda v: to_temp(struct.unpack('<h', v)[0]), 278 1: UNPACK_DECO, 279 2: lambda v: None, 280 3: lambda v: None, 281 4: lambda v: None, 282 5: lambda v: None, 283 6: lambda v: None, 284 } 285 # determine parser for each divisor type 286 parsers = [div_type_parser[d.type] for d in profile_header.divs] 287 288 def parser(data): 289 # split data into extended information items 290 items = partition(data, *idx) 291 # parse each extended information item 292 return tuple(f(v) if v else None for f, v in zip(parsers, items))
293 294 return parser 295
296 -def model_version(data):
297 """ 298 Get model and version information about a hwOS family OSTC dive 299 computer. 300 301 :param data: Raw data fetched from a hwOS family OSTC dive computer. 302 303 .. seealso:: `RawData` 304 """ 305 major, minor = data.version[2:4] 306 dsc = data.features[1] 307 model = MODEL.get(dsc, 'OSTC hwOS') 308 logger.debug('descriptor 0x{:x} -> model {}'.format(dsc, model)) 309 return model, major, minor
310
311 -def parse_header(data):
312 """ 313 Parse dive header data read from hwOS OSTC dive computer. 314 """ 315 parsers = ( 316 identity, # start marker 317 to_int, to_timestamp, to_depth, to_duration, to_temp, 318 to_gas_list, 319 # average depth, dive mode 320 to_depth, identity, 321 identity # end marker 322 ) 323 # 5s - datetime, 20s - gas list, B - dive mode 324 fmt = '<H 7x 3s 5s H 3s h 4x 20s 25x H 7x B 171x H' 325 fields = header_fields(parsers, fmt, data) 326 return Header._make(fields)
327
328 -def parse_profile_header(data):
329 """ 330 Parse dive profile header (aka small header). 331 332 :param data: Dive profile data. 333 """ 334 size, rate, no_div = struct.unpack('3sBB', data[:5]) 335 size = to_int(size) 336 337 divs = data[5:5 + 3 * no_div] 338 assert len(divs) == 3 * no_div 339 divs = struct.unpack('BBB' * no_div, divs) 340 divs = itz.partition(3, divs) 341 divs = tuple(Divisor._make(v) for v in divs) 342 343 return ProfileHeader(size, rate, divs)
344
345 -def dive_profile_size(data):
346 """ 347 Extract dive profile size from dive header raw data. 348 """ 349 parsers = (identity, to_int, identity) 350 fmt = '<H7x3s242xH' 351 data = header_fields(parsers, fmt, data) 352 return data[0]
353
354 -def dive_headers_data(headers):
355 """ 356 Divide all dive header raw data into collection of raw data for each 357 header. 358 359 :param headers: Raw header data fetched from hwOS OSTC dive computer. 360 """ 361 assert len(headers) == 65536 362 363 items = itz.partition(256, headers) 364 # convert back to bytes, see 365 # 366 # - https://github.com/pytoolz/cytoolz/issues/102 367 # - https://github.com/pytoolz/toolz/issues/377 368 # 369 # also filter unused headers via `logbook-profile version` header field 370 yield from (bytes(v) for v in items if v[8] != 0xff)
371
372 -def header_fields(parsers, fmt, data):
373 """ 374 Extract dive header fields data from dive header raw data. 375 376 :param parsers: Parser for each field, including start and end markers. 377 :param fmt: Struct format to extract the fields. 378 :param data: Dive header raw data. 379 """ 380 assert struct.calcsize(fmt) == 256, struct.calcsize(fmt) 381 382 items = struct.unpack(fmt, data) 383 items = zip(parsers, items) 384 start, *fields, end = (p(v) for p, v in items) 385 assert start == 0xfafa and end == 0xfbfb, '0x{:x} 0x{:x}'.format(start, end) 386 387 return fields
388
389 -def dive_profile_sample_idx(data):
390 """ 391 Determine index of each dive profile sample using the profile raw data. 392 393 :param data: Dive profile raw data. 394 """ 395 # there is no more than `n_samples` dive profile samples 396 n_samples = len(data) // MIN_SAMPLE_SIZE 397 398 f = partial(dive_profile_next_sample, data) 399 idx = itz.accumulate(f, range(n_samples)) 400 idx = takewhile(lambda i: i < len(data), idx) 401 return idx
402
403 -def dive_profile_next_sample(data, idx, sample_no):
404 """ 405 Calculate index of next dive profile sample. 406 407 The calculation is performed using dive profile raw data and index of 408 current dive profile sample. The `idx + 2` points to profile flag byte, 409 which is used to determine total length of current sample. 410 411 :param data: Dive profile raw data. 412 :param idx: Index of current dive profile sample. 413 :param sample_no: Sample number (unused). 414 """ 415 return idx + (data[idx + 2] & 0x7f) + MIN_SAMPLE_SIZE
416
417 -def partition(data, *idx):
418 """ 419 Partition the data using indexes. 420 421 Each index is start of each item. 422 423 :param data: Data to partition. 424 :param idx: Indexes used to partition the data. 425 """ 426 item_range = idx + (None,) 427 item_range = zip(item_range[:-1], item_range[1:]) 428 yield from (data[j:k] for j, k in item_range)
429
430 -def create_dive(header, data):
431 """ 432 Create Kenozooid dive record. 433 434 :param header: Dive header of a dive stored in hwOS OSTC dive computer. 435 :param data: Dive profile raw data. 436 """ 437 return kd.Dive( 438 datetime=header.datetime, 439 depth=header.depth, 440 duration=header.duration, 441 temp=header.temp, 442 avg_depth=header.avg_depth, 443 mode=DIVE_MODES[header.mode], 444 profile=list(parse_profile(header, data)), 445 )
446
447 -def create_sample(header, profile_header, ext_parser, sample_no, data):
448 """ 449 Create Kenozooid dive profile sample record. 450 451 :param header: Dive header. 452 :param profile_header: Dive profile header. 453 :param ext_parser: Extended information parser. 454 :param sample_no: Dive profile sample number. 455 :param data: Dive profile sample data. 456 """ 457 depth = to_depth_adj(to_int(data[:2])) 458 time = sample_no * profile_header.rate 459 460 events = parse_events(data) 461 462 gas_no = events.events.gas 463 assert gas_no is None or gas_no > 0 464 gas = header.gas_list[gas_no - 1][0] if events.events.gas else None 465 # probably not possible, but overwrite current gas mix with the mix set 466 # manually 467 gas = events.events.manual_gas if events.events.manual_gas else gas 468 469 start = sum(bool(v) * s for v, s in zip(events.events, EVENT_DATA_SIZE)) 470 start += MIN_SAMPLE_SIZE # include depth and profile byte 471 472 # parse extended information 473 temp, deco, *_ = ext_parser(data[start:]) 474 deco_depth, deco_time = deco if deco and deco[0] else (None, None) 475 476 if __debug__ and gas: 477 logger.debug('Dive {}, gas number: {}, gas: {}'.format( 478 header.datetime, events.events.gas, gas 479 )) 480 481 alarm = ('deco',) if events.alarm == 2 else None 482 return kd.Sample( 483 depth=depth, 484 time=time, 485 temp=temp, 486 deco_time=deco_time, 487 deco_depth=deco_depth, 488 alarm=alarm, 489 gas=gas, 490 )
491 492 # vim: sw=4:et:ai 493