Source Code for Module mobi.mtld.da.device.tree

  1  #!/usr/bin/env python 
  2   
  3  """ 
  4  @copyright: 
  5   Copyright (c) 2014 by mTLD Top Level Domain Limited. All rights reserved.\n 
  6   Portions copyright (c) 2008 by Argo Interactive Limited.\n 
  7   Portions copyright (c) 2008 by Nokia Inc.\n 
  8   Portions copyright (c) 2008 by Telecom Italia Mobile S.p.A.\n 
  9   Portions copyright (c) 2008 by Volantis Systems Limited.\n 
 10   Portions copyright (c) 2002-2008 by Andreas Staeding.\n 
 11   Portions copyright (c) 2008 by Zandan.\n 
 12  @author: dotMobi 
 13  """ 
 14   
 15  import re 
 16  try: 
 17   import json as simplejson 
 18  except ImportError: 
 19   import simplejson 
 20   
 21  from mobi.mtld.da.device.ua_props import UaProps 
 22  from mobi.mtld.da.device.client_props import ClientProps 
 23  from mobi.mtld.da.data_type import DataType 
 24  from mobi.mtld.da.property import Property 
 25  from mobi.mtld.da.properties import Properties 
 26  from mobi.mtld.da.exception.data_file_exception import DataFileException 
 27  from mobi.mtld.da.exception.client_properties_exception import ClientPropertiesException 
 28   
 29 -class Tree(object): 
 30   
 31   MIN_JSON_VERSION = 0.7 
 32   
 33   # for MAP optimization, set to average number properties 
 34   MAP_INITIAL_CAPACITY = 100 
 35   
 36   # device-id property name 
 37   KEY_DEVICE_ID = 'iid' 
 38   
 39   # tree key - {DATA-META-DATA} 
 40   KEY_META = '$' 
 41   
 42   # tree key - tree structure version 
 43   KEY_META_VERSION = 'Ver' 
 44   
 45   # tree key - tree revision 
 46   KEY_META_REVISION = 'Rev' 
 47   
 48   # tree key - tree create time-stamp 
 49   KEY_META_TIMESTAMP = 'Utc' 
 50   
 51   # tree key - property names [property-name,] 
 52   KEY_PROPERTY_NAMES = 'p' 
 53   
 54   # tree key - property values [property-value,] 
 55   KEY_VALUES = 'v' 
 56   
 57   # tree key - list of regexes [regex(String),] 
 58   KEY_REGEX = 'r' 
 59   
 60   # tree key - list of compiled regexes, the regexes are compiled and put inside 
 61   # [regex(Pattern),] 
 62   KEY_COMPILED_REGEX = 'creg' 
 63   
 64   # tree key - user-agent header names 
 65   # {"h":{"sl":["x-device-user-agent","x-original-user-agent"]}} 
 66   KEY_HEADERS = 'h' 
 67   
 68   # tree key - stock user-agent header names 
 69   # {"h":{"sl":["x-device-user-agent","x-original-user-agent"]}} 
 70   KEY_UA_STOCK_HEADERS = 'sl' 
 71   
 72   # tree key - tree main branch 
 73   KEY_MAIN = 't' 
 74   
 75   # tree key - property data 
 76   KEY_DATA = 'd' 
 77   
 78   # tree key - has children 
 79   KEY_CHILDREN = 'c' 
 80   
 81   # tree key - masked properties 
 82   KEY_MASKED = 'm' 
 83   
 84   properties = None 
 85   tree = None 
 86   data_revision = None 
 87   
 88   # A list of http-headers which may contain the original user-agent. 
 89   # if the tree does not contain KEY_UA_STOCK_HEADERS then this list will be 
 90   # used 
 91   stock_ua_headers = [] 
 92   
 93   __config = None 
 94   __ua_props = None 
 95   __client_props = None 
 96   __device_id_prop_name_id = None 
 97   
 98   
 99 - def __init__(self, json, config): 
100    """ 
101    Load the JSON tree into a dict. 
102    """ 
103    self.__config = config 
104   
105    self.tree = simplejson.loads(json) 
106   
107    if self.tree == {}: 
108     raise DataFileException('Unable to load JSON data') 
109   
110    if self.KEY_META not in self.tree: 
111     raise DataFileException('Bad data loaded into the tree') 
112   
113    if (float(self.tree[self.KEY_META][self.KEY_META_VERSION]) < 
114    self.MIN_JSON_VERSION): 
115     raise DataFileException('DeviceAtlas JSON file must be version 0.7 or ' + 
116     'later. Please download a more recent version') 
117   
118    # Prepare the user-agent rules branch before we start recognition. 
119    # To maintain backwards compatibility - only do this if we have the ua rules 
120    # branch 
121    if UaProps.KEY_UA_RULES in self.tree and self.__config.include_ua_props: 
122     self.__ua_props = UaProps(self) # stick in the tree so we can use it later 
123     # remove the UAR branch to save some memory 
124    elif UaProps.KEY_UA_RULES in self.tree: 
125     del self.tree[UaProps.KEY_UA_RULES] 
126   
127    # Prepare client side properties. 
128    if ClientProps.KEY_CP_RULES in self.tree: 
129     self.__client_props = ClientProps(self) 
130   
131    # cache values from the tree which are used by the API 
132    property_ids = self.tree[self.KEY_PROPERTY_NAMES] 
133   
134    i = 0 
135    for property_id in property_ids: 
136     if self.KEY_DEVICE_ID == property_id: 
137      self.__device_id_prop_name_id = str(i) 
138      break 
139     i += 1 
140   
141    # set ua headers 
142    self.stock_ua_headers = [ 
143     "x-device-user-agent", 
144     "x-original-user-agent", 
145     "x-operamini-phone-ua", 
146     "x-skyfire-phone", 
147     "x-bolt-phone-ua", 
148     "device-stock-ua", 
149     "x-ucbrowser-ua", 
150     "x-ucbrowser-device-ua", 
151     "x-ucbrowser-device", 
152     "x-puffin-ua" 
153    ] 
154   
155    # update stock user-agent headers from tree 
156    if self.KEY_HEADERS in self.tree and self.tree[self.KEY_HEADERS] is not None: 
157     ua_stock_headers = self.tree[self.KEY_HEADERS][self.KEY_UA_STOCK_HEADERS] 
158     if ua_stock_headers is not None: 
159      self.stock_ua_headers = ua_stock_headers 
160   
161    # set data revision 
162    self.data_revision = self.tree[self.KEY_META][self.KEY_META_REVISION] 
163   
164 - def property_names(self): 
165    """ 
166    Get the list of all available property names from the tree (not contains 
167    client side props) 
168    """ 
169    return self.tree[self.KEY_PROPERTY_NAMES] 
170   
171 - def data_version(self): 
172    """ 
173    Get data file version. 
174    """ 
175    return self.tree[self.KEY_META][self.KEY_META_VERSION] 
176   
177 - def data_creation_timestamp(self): 
178    """ 
179    Get data file creation timestamp. 
180    """ 
181    return self.tree[self.KEY_META][self.KEY_META_TIMESTAMP] 
182   
183 - def put_properties(self, user_agent, stock_user_agents, 
184    client_side_properties = None): 
185    """ 
186    Get properties from tree walk/ua/client-side and put them in the 
187    tree.properties 
188   
189    @param user_agent: user-agent string (from the original User-Agent header) to be 
190    used for detecting ua-props 
191    @param stock_user_agents: list of candidate user-agent strings to be used for 
192    tree walk 
193    @param client_side_properties: optional client side properties 
194    """ 
195   
196    self.properties = Properties() 
197   
198    self.put_tree_walk_properties(user_agent, stock_user_agents) 
199   
200    if client_side_properties is not None and client_side_properties != "": 
201     if self.__client_props is None: 
202      # stop if the JSON file does not contain the required CPR section 
203      raise ClientPropertiesException('JSON data does not support client ' + 
204      'properties.') 
205     self.__client_props.put_properties(client_side_properties) 
206   
207 - def put_tree_walk_properties(self, user_agent, stock_user_agents = None): 
208    """ 
209    Get properties from tree walk/ua and put them in the tree.properties 
210   
211    if stock_user_agents is not None: 
212     - iterate over stock_user_agents 
213       for each item: tree-walk and stop iteration if result has deviceId 
214     - use userAgent for detecting the ua-props 
215   
216    if stock_user_agents is None: 
217     - use userAgent for tree walk 
218     - use userAgent for detecting the ua-props 
219   
220    @param user_agent: user-agent string (from the original User-Agent header) 
221    @param stock_user_agents: list of candidate user-agent strings to be used for 
222    tree walk 
223    """ 
224   
225    include_ua_props = self.__config.include_ua_props 
226   
227    # props_to_vals = {property-id-from-tree-p: value-id-from-tree-v,} 
228    props_to_vals = {} 
229    regexes = self.tree[self.KEY_REGEX][str(UaProps.API_ID)] 
230    tree_main = self.tree[self.KEY_MAIN] 
231    matched = "" 
232   
233    # Remove spaces and backslashes 
234    user_agent = user_agent.strip().replace("\/", "/") 
235   
236    if stock_user_agents is None: 
237   
238     self.__seek_properties(tree_main, user_agent, props_to_vals, matched, 
239      regexes) 
240   
241    else: 
242   
243     for stock_user_agent in stock_user_agents: 
244      stock_user_agent = stock_user_agent.replace("\/", "/") 
245      self.__seek_properties(tree_main, stock_user_agent, props_to_vals, matched, 
246      regexes) 
247      if self.__device_id_prop_name_id in props_to_vals: 
248       break 
249   
250    # put the detected properties which are as 
251    # {property-id-from-tree-p: value-id-from-tree-v,} 
252    # into the (Properties) properties object 
253    for property_id, value_id in props_to_vals.items(): 
254     name = self.property_name_by_id(int(property_id)) 
255     self.properties[name[1:]] = Property(self.property_value_by_id(value_id), 
256      name[0]) 
257   
258    # matched and un-matched 
259    if self.__config.include_match_info: 
260     # add in matched and unmatched UA parts 
261     self.properties['_matched'] = Property(matched, DataType.STRING) 
262     self.properties['_unmatched'] = Property(user_agent[len(matched):], 
263      DataType.STRING) 
264   
265    # get ua-props from the original user-agent header 
266    if include_ua_props and self.__ua_props is not None: 
267     self.__ua_props.put_properties(user_agent, props_to_vals) 
268   
269   
270 - def property_name_by_id(self, property_id): 
271    return self.tree[self.KEY_PROPERTY_NAMES][property_id] 
272   
273 - def property_value_by_id(self, value_id): 
274    return self.tree[self.KEY_VALUES][value_id] 
275   
276   # Private 
277   
278 - def __seek_properties(self, node, string, props_to_vals, matched, regex_rules, 
279    sought = None): 
280   
281    if self.KEY_DATA in node and node[self.KEY_DATA] is not None: 
282     data = node[self.KEY_DATA] 
283     if sought is None: 
284      for key, value in data.items(): 
285       props_to_vals[key] = value 
286   
287     else: 
288      for name_id in sought: 
289       value_id = data[name_id] 
290       if value_id is not None: 
291        props_to_vals[name_id] = value_id 
292        if self.KEY_MASKED not in node or name_id not in node[self.KEY_MASKED]: 
293         del sought[name_id] 
294         if sought is None or len(sought) == 0: 
295          return 
296   
297    if self.KEY_CHILDREN in node: 
298     if self.KEY_REGEX in node: 
299      to_run = node[self.KEY_REGEX] 
300      max_regex_rules = len(regex_rules) 
301      for to_run_j in to_run: 
302       if to_run_j < max_regex_rules: 
303        patt = regex_rules[to_run_j] 
304        string = re.sub(patt, '', string) 
305     # recursively walk the tree 
306     if string is not None: 
307      length = len(string) + 1 
308      children = node[self.KEY_CHILDREN] 
309      for k in range(0, length): 
310       seek = string[0:k] 
311       if seek in children: 
312        matched += seek 
313        return self.__seek_properties(children[seek], string[k:], props_to_vals, 
314         matched, regex_rules, sought) 
315