mobi.mtld.da.device.ua

20 """ 21 This class tries to extract properties from the User-Agent string itself. This 22 is a completely separate step to the main JSON tree walk but uses the results 23 of the tree walk to optimise the property extraction. The property extraction 24 is done in two steps. 25 26 Step 1: Try and identify the type of User-Agent and thus the set of property 27 extraction rules to run. This is optimised by the properties from the tree 28 walk. 29 30 Step 2: Run the rules found in step 1 to try and extract the properties. 31 """ 32 33 API_ID = 5 34 35 # Constants to make keys more readable 36 KEY_UA_RULES = 'uar' 37 KEY_SKIP_IDS = 'sk' 38 KEY_DEFAULT_REGEX_SET = 'd' 39 KEY_RULE_GROUPS = 'rg' 40 KEY_RULE_REGEX_ID = 'r' 41 KEY_REGEXES = 'reg' 42 KEY_REGEX_MATCH_POS = 'm' 43 KEY_REFINE_REGEX_ID = 'f' 44 KEY_SEARCH_REGEX_ID = 's' 45

46 - def __init__(self, tree_provider):

47 super(UaProps, self).__init__(tree_provider, self.KEY_UA_RULES) 48 # process the regexes - we need to override the default ones with any API 49 # specific regexes 50 self.__init_process_regexes()

51

52 - def put_properties(self, user_agent, props_to_vals, sought = None):

53 # first check list of items that skip rules - these are typically non-mobile 54 # boolean properties such as isBrowser, isBot etc 55 56 if self.__skip_ua_rules(props_to_vals): 57 return 58 59 regexes = self.branch[self.KEY_REGEXES] 60 61 # now find the rules to run on the UA. This is a two step process. 62 # Step 1 identifies the UA type and finds as list of rules to run. 63 # Step 2 uses the list of rules to find properties in a UA 64 65 # STEP 1: try and find the rules to run on the UA 66 rule_groups = self.branch[self.KEY_RULE_GROUPS] 67 rules_to_run = self.__ua_property_rules(user_agent, props_to_vals, 68 rule_groups, regexes) 69 70 # STEP 2: try and extract properties using the rules 71 if rules_to_run is not None: 72 self.__extract_properties(rules_to_run, user_agent, regexes, sought)

73 74 # Protected 75

76 - def _init_get_matcher_propery_ids(self, group, prop_ids):

77 """ 78 Find all the properties that are used for matching. 79 80 @param group: The rule group that can contain a property matcher 81 @param prop_ids: The list of found property IDs 82 """ 83 # the properties matcher may not exist.... 84 if group[self.KEY_PROPERTY_MATCHER]: 85 for prop_id, prop_value in group[self.KEY_PROPERTY_MATCHER].items(): 86 if prop_id not in prop_ids: 87 prop_ids.append(prop_id)

88

89 - def _init_rule_sets(self, group):

90 """ 91 Prepare the rule set by extracting it from the current group and counting 92 the items in the group. This is done to avoid counting the items on every 93 request. 94 95 @param group: The current parent group. 96 """ 97 sets = group[self.KEY_RULE_SET] 98 group[self.KEY_RULE_SET_COUNT] = len(sets) 99 return sets

100 101 # Private 102

103 - def __init_process_regexes(self):

104 """ 105 Process the regexes by overriding any default ones with API specific regexes 106 and then compile the list of regexes. This also changes the regex key from a 107 string to an integer for easier retrieval later on. 108 """ 109 # process regexes... 110 # override default regexes if we have API specific ones to use 111 all_reg = self.branch[self.KEY_REGEXES] 112 default_reg = all_reg[self.KEY_DEFAULT_REGEX_SET] 113 114 if self.API_ID in all_reg: 115 # now loop over all api reg and replace the default regexes 116 for key, reg_item in all_reg[self.API_ID].items(): 117 default_reg[key] = reg_item 118 119 # no need to compile regexes 120 121 # save some memory and remove the nodes we won't use by setting the main 122 # regex node to be the new regex list 123 self.branch[self.KEY_REGEXES] = default_reg

124

125 - def __extract_properties(self, rules_to_run, user_agent, regexes, sought):

126 """ 127 This function loops over all the rules in rules_to_run and returns any 128 properties that match. The properties returned can be typed or strings. 129 130 @param rules_to_run: The rules to run against the User-Agent to find the 131 properties. 132 @param user_agent: The User-Agent to find properties for. 133 @param regexes: The list of compiled regexes. 134 @param sought: A set of properites to return values for. 135 """ 136 137 # Loop over the rules array, each object in the array can contain 4 items: 138 # propertyid, propertyvalue, regexid and regexmatch_position 139 for rule_details in rules_to_run: 140 141 ruleprop_id = rule_details[self.KEY_PROPERTY_MATCHER] 142 143 # check if we are looking for a specific property, if so and the 144 # current rule property id is not it then continue 145 if (sought is not None) and (ruleprop_id not in sought): 146 continue 147 148 prop_name = self.tree_provider.property_name_by_id(ruleprop_id) 149 150 # do we have a property we can set without running the regex rule? 151 if self.KEY_PROPERTY_VALUE in rule_details: 152 153 # we have an ID to the value... 154 prop_val_id = rule_details[self.KEY_PROPERTY_VALUE] 155 value = self.tree_provider.property_value_by_id(prop_val_id) 156 157 self.tree_provider.properties[prop_name[1:]] = Property(value, prop_name[0]) 158 159 else: 160 161 # otherwise apply the rule to extract the property from the UA 162 regex_id = rule_details[self.KEY_RULE_REGEX_ID] 163 patt = regexes[regex_id] 164 165 matches = re.search(patt, user_agent) 166 167 if matches is not None: 168 match_pos = rule_details[self.KEY_REGEX_MATCH_POS] 169 match_res = matches.group(match_pos) 170 171 if match_res != "": 172 173 value = match_res 174 self.tree_provider.properties[prop_name[1:]] = Property(value, 175 prop_name[0])

176 177 # end else 178 179 # end for 180 181 # end method 182

183 - def __skip_ua_rules(self, id_properties):

184 """ 185 Check list of items that skip rules - these are typically non-mobile boolean 186 properties such as isBrowser, isBot, isCrawler, etc. 187 188 @param id_properties: The results of the tree walk, map of property id to 189 value id 190 @return: TRUE if the UA rules are to be skipped, FALSE if they have to be run 191 """ 192 skip_list = self.branch[self.KEY_SKIP_IDS] 193 194 for prop_id in skip_list: 195 196 str_prop_id = str(prop_id) 197 198 if str_prop_id in id_properties: 199 prop_val = self.tree_provider.property_value_by_id(id_properties[str_prop_id]) 200 if prop_val is not None and prop_val != 0: # 2nd condition is v. important 201 return True 202 return False

203

204 - def __ua_property_rules(self, user_agent, id_properties, rule_groups, regexes):

205 """ 206 Try and find a set of property extraction rules to run on the User-Agent. 207 This is done in two ways. 208 209 The first way uses properties found from the tree walk to identify the 210 User-Agent type. If there are still multiple UA types then refining regexes 211 can be run. 212 213 If the above approach fails to find a match then fall back to the second way 214 which uses a more brute regex search approach. 215 216 Once the UA type is known the correct set of property extraction rules can 217 be returned. 218 219 @param user_agent: The User-Agent to find properties for. 220 @param id_properties: The results of the tree walk, map of property id to value id. 221 @param rule_groups: All the rule groups that contain the matchers and the rules to 222 run. 223 @param regexes: The list of compiled regexes. 224 @return: a map of rules to run against the User-Agent or None if no rules 225 are found. 226 """ 227 228 rules_to_run = [] 229 230 # Method one - use properties from tree walk to speed up rule search 231 rules_to_run = self.__find_rules_by_properties(rule_groups, user_agent, 232 id_properties, regexes) 233 234 # No match found using the properties so now we loop over all rule groups 235 # again and try to use a more brute force attempt to find the rules to run 236 # on this user-agent. 237 238 # continue to find extra rules even if we found rules from the property 239 # matcher 240 temp_rules = self.__find_rules_by_regex(rule_groups, user_agent, regexes) 241 if temp_rules is not None and len(temp_rules) > 0: 242 rules_to_run = rules_to_run + temp_rules # << Can be optimized 243 244 return rules_to_run

245

246 - def __find_rules_by_properties(self, groups, user_agent, id_properties, 247 regexes):

248 """ 249 Try and find User-Agent type and thus the rules to run by using the 250 properties returned from the tree walk. All the properties defined in the 251 property matcher set must match. If a match is found then the rules can be 252 returned. 253 254 @param groups: The rule groups to loop over. 255 @param user_agent: The User-Agent to find properties for. 256 @param regexes: The list of compiled regexes. 257 @return: a dict of rules to run against the User-Agent or None if no 258 rules are found. 259 """ 260 261 rules_to_run_a = [] 262 263 if groups is None or len(groups) == 0: 264 return rules_to_run_a 265 266 for group in groups: 267 268 # check if we have the property match list 269 if group is None or group[self.KEY_PROPERTY_MATCHER] is None: 270 continue 271 272 # try matching defined properties so we know what rules to run. If there 273 # is a match then we can return the rules to run. In some cases we need to 274 # refine the match found by running some refining regexes 275 prop_match = self.__check_properties_match(group[self.KEY_PROPERTY_MATCHER], 276 id_properties) 277 278 if prop_match: 279 rule_set = group[self.KEY_RULE_SET] 280 rule_set_count = group[self.KEY_RULE_SET_COUNT] 281 282 # in some cases we have multiple rule_sets to choose from, if more 283 # than 1 we need to run some additional refining regex rules. 284 285 if rule_set_count > 1: 286 rules_to_run = self.__find_rules_to_run_by_regex(user_agent, rule_set, 287 rule_set_count, regexes, self.KEY_REFINE_REGEX_ID) 288 else: 289 rules_set = rule_set[0] # 0th item... there should only be one... 290 rules_to_run = rules_set[self.KEY_RULE_ARR] 291 292 if len(rules_to_run) > 0: 293 rules_to_run_a = rules_to_run_a + rules_to_run # << Can be optimized 294 295 return rules_to_run_a

296

297 - def __check_properties_match(self, prop_list, props_to_values):

298 """ 299 This functions checks all the properties in the property matcher branch of 300 this rule group. This branch contains a list of properties and their values. 301 All must match for this function to return true. 302 303 In reality the properties and values are indexes to the main property and 304 value arrays. 305 306 @param prop_list: The list of properties to check for matches. 307 @param props_to_values: Dict of property and value ids 308 @return: TRUE if ALL properties match, false otherwise. 309 """ 310 prop_match = False 311 312 # loop over prop_list and try and match ALL properties 313 for prop_id, expected_value_id in prop_list.items(): 314 315 # get the value found via the tree walk 316 if prop_id in props_to_values: 317 318 tree_value_id = props_to_values[prop_id] 319 320 # we can speed things up a little by just comparing the IDs! 321 if tree_value_id == expected_value_id: 322 prop_match = True # no break here as we want to check all properties 323 else: 324 # there was code here to check actual values if the IDs did not match 325 # but is was unnecessary. If the JSON generator is working correctly then 326 # just the ID check is sufficient. 327 return False 328 329 else: 330 return False 331 332 return prop_match

333

334 - def __find_rules_by_regex(self, groups, user_agent, regexes):

335 """ 336 Search for the rules to run by checking the User-Agent with a regex. If 337 there is a match the rule list is returned. 338 339 @param groups: The rule groups to loop over. 340 @param user_agent: The User-Agent to find properties for. 341 @param regexes: The list of compiled regexes. 342 @return: a dict of rules to run against the User-Agent or nil if no rules 343 are found. 344 """ 345 rules_to_run = [] 346 347 if len(groups) > 0: 348 349 i = 0 350 n = len(groups) 351 352 while i < n and len(rules_to_run) > 0: 353 354 group = groups[i] 355 rules_to_run = self.__find_rules_to_run_by_regex( 356 user_agent, 357 group[self.KEY_RULE_SET], 358 group[self.KEY_RULE_SET_COUNT], 359 regexes, 360 self.KEY_SEARCH_REGEX_ID) 361 362 i += 1 363 364 return rules_to_run

365 366

367 - def __find_rules_to_run_by_regex(self, user_agent, rule_set, rule_set_count, 368 regexes, type):

369 """ 370 Loop over a set of refining rules to try and determine the User-Agent type 371 and so find the rules to run on it. 372 373 @param user_agent: The User-Agent to find properties for. 374 @param rule_set: The rule_set that contains the search regex id, refine regex id 375 and the magical rules_to_run. 376 @param rule_set_count: The pre-counted items in rule_set. 377 @param regexes: The list of compiled regexes. 378 @param type: The type of rule to run either Refine or Search. 379 @return: a dict of rules to run against the User-Agent or nil if no rules 380 are found. 381 """ 382 383 # we want these to run in the order they appear. For some reason the Json 384 # class uses a Hash to represent an array of items so we have to loop 385 # based on the index of the Hash 386 387 for i in range(0, rule_set_count): 388 389 set = rule_set[i] 390 391 # get refine / search id to run 392 if type in set: 393 394 regex_id = set[type] 395 396 # now look up the pattern... 397 regex = regexes[regex_id] 398 399 if re.search(regex, user_agent): 400 return set[self.KEY_RULE_ARR] # now get the rules to run! 401 402 return []

Source Code for Module mobi.mtld.da.device.ua_props