From 62b560ad8e4a697761de835467c98805ca776b55 Mon Sep 17 00:00:00 2001 From: Django Faiola <157513033+djangofaiola@users.noreply.github.com> Date: Sat, 3 May 2025 10:34:05 +0200 Subject: [PATCH 1/4] Update BeReal.py Added support for Hackemist SDWebImage Cache, EntitiesStore.sqlite, Cache.db Fixed minor bugs --- scripts/artifacts/BeReal.py | 4133 +++++++++++++++++++++++++++-------- 1 file changed, 3270 insertions(+), 863 deletions(-) diff --git a/scripts/artifacts/BeReal.py b/scripts/artifacts/BeReal.py index 7d352ae9..87dadf2f 100644 --- a/scripts/artifacts/BeReal.py +++ b/scripts/artifacts/BeReal.py @@ -1,70 +1,76 @@ __artifacts_v2__ = { "bereal_preferences": { - "name": "BeReal Preferences", + "name": "Preferences", "description": "Parses and extract BeReal Preferences", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-09", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", - "paths": ('*/mobile/Containers/Shared/AppGroup/*/Library/Preferences/group.BeReal.plist'), + "paths": ('*/mobile/Containers/Shared/AppGroup/*/Library/Preferences/group.BeReal.plist', + '*/mobile/Containers/Data/Application/*/Library/Preferences/AlexisBarreyat.BeReal.plist'), "output_types": [ "none" ], "artifact_icon": "settings" }, "bereal_accounts": { - "name": "BeReal Accounts", + "name": "Accounts", "description": "Parses and extract BeReal Accounts", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", - "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-ProfileRepository/*'), + "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-ProfileRepository/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Profile picture URL", "Timezone", "Device UID", "Device Model and OS", "App version", "RealMojis" ], + "html_columns": [ "Profile picture URL", "Timezone", "Device UID", "Device Model and OS", "App version", "RealMojis", + "Source file name", "Location" ], "artifact_icon": "user" }, "bereal_contacts": { - "name": "BeReal Contacts", + "name": "Contacts", "description": "Parses and extract BeReal Contacts", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2025-04-02", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-RelationshipsContactsManager-contact/*'), "output_types": [ "lava", "html", "tsv" ], - "html_columns": [ "Profile picture"], + "html_columns": [ "Source file name", "Location" ], "artifact_icon": "users" }, "bereal_persons": { - "name": "BeReal Persons", + "name": "Persons", "description": "Parses and extract BeReal Persons", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", - "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/PersonRepository/*'), + "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/PersonRepository/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_officialAccountProfiles/*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Profile picture URL", "Urls" ], + "html_columns": [ "Profile picture URL", "URLs", "Source file name", "Location" ], "artifact_icon": "users" }, "bereal_friends": { - "name": "BeReal Friends", + "name": "Friends", "description": "Parses and extract BeReal Friends, Friend Requests Sent, Friend Requests Received, and Friends Following", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", @@ -72,74 +78,82 @@ '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-RelationshipsRequestSentListManager/*', '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-RelationshipsRequestReceivedListManager/*', '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-Production_FriendsStorage.following/*', - '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-Production_FriendsStorage.followers/*'), + '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-Production_FriendsStorage.followers/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Profile picture URL" ], + "html_columns": [ "Profile picture URL", "Source file name", "Location" ], "artifact_icon": "user-plus" }, "bereal_blocked_users": { - "name": "BeReal Blocked Users", + "name": "Blocked Users", "description": "Parses and extract BeReal Blocked Users", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", - "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-BlockedUserManager/*'), + "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-BlockedUserManager/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Profile picture URL" ], + "html_columns": [ "Source file name", "Location" ], "artifact_icon": "slash" }, "bereal_posts": { - "name": "BeReal Posts", + "name": "Posts", "description": "Parses and extract BeReal Memories, Person BeReal of the day and Production Feeds", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-MemoriesRepository-subject-key/*', '*/mobile/Containers/Data/Application/*/Library/Caches/PersonRepository/*', - '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*'), + '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Primary URL", "Secondary URL", "Thumbnail URL", "Tagged friends", "Source file name", "Location" ], + "html_columns": [ "Primary URL", "Secondary URL", "Thumbnail URL", "Song URL", "Visibility", "Tagged friends", "Source file name", "Location" ], "artifact_icon": "calendar" }, "bereal_pinned_memories": { - "name": "BeReal Pinned Memories", + "name": "Pinned Memories", "description": "Parses and extract BeReal Pinned Memories", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-MemoriesRepository-pinnedMemories-key/*', - '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-PersonRepository-pinnedMemories-key/*'), + '*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-PersonRepository-pinnedMemories-key/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Primary URL", "Secondary URL", "Thumbnail URL", "Source file name", "Location" ], + "html_columns": [ "Primary URL", "Secondary URL", "Source file name", "Location" ], "artifact_icon": "bookmark" }, "bereal_realmojis": { - "name": "BeReal RealMojis", + "name": "RealMojis", "description": "Parses and extract BeReal RealMojis from my memories and Person's memories", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-MemoriesRepository-subject-key/*', '*/mobile/Containers/Data/Application/*/Library/Caches/PersonRepository/*', - '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*'), + '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "RealMoji" ], + "html_columns": [ "RealMoji URL", "Source file name", "Location" ], "chatParams": { "timeColumn": "Created", "threadDiscriminatorColumn": "BeReal ID", @@ -152,20 +166,22 @@ "artifact_icon": "thumbs-up" }, "bereal_comments": { - "name": "BeReal Comments", + "name": "Comments", "description": "Parses and extract BeReal Comments from my memories, Person's posts, and Production post Feeds", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Library/Caches/disk-bereal-MemoriesRepository-subject-key/*', '*/mobile/Containers/Data/Application/*/Library/Caches/PersonRepository/*', - '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*'), + '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*', + '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', + '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "RealMojis" ], + "html_columns": [ "RealMojis", "Source file name", "Location" ], "chatParams": { "timeColumn": "Created", "threadDiscriminatorColumn": "BeReal ID", @@ -178,18 +194,18 @@ "artifact_icon": "message-square" }, "bereal_messages": { - "name": "BeReal Messages", + "name": "Messages", "description": "Parses and extract BeReal Messages", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Shared/AppGroup/*/bereal-chat.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Media URL" ], + "html_columns": [ "Media URL", "Source file name", "Location" ], "chatParams": { "timeColumn": "Sent", "threadDiscriminatorColumn": "Thread ID", @@ -202,18 +218,18 @@ "artifact_icon": "message-square" }, "bereal_chat_list": { - "name": "BeReal Chat List", + "name": "Chat List", "description": "Parses and extract BeReal Chat List", "author": "@djangofaiola", - "version": "0.1", + "version": "0.2", "creation_date": "2024-12-20", - "last_update_date": "2024-03-08", + "last_update_date": "2025-05-03", "requirements": "none", "category": "BeReal", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Shared/AppGroup/*/bereal-chat.sqlite*'), "output_types": [ "lava", "html", "tsv", "timeline" ], - "html_columns": [ "Administrators", "Participants" ], + "html_columns": [ "Administrators", "Participants", "Source file name", "Location" ], "artifact_icon": "message-circle" } } @@ -221,32 +237,30 @@ from pathlib import Path import inspect import json -from datetime import timedelta +import sqlite3 +import re +import hashlib +from datetime import timedelta, datetime from base64 import standard_b64decode from urllib.parse import urlparse, urlunparse -from scripts.ilapfuncs import get_file_path, get_sqlite_db_records, get_plist_content, get_plist_file_content, convert_unix_ts_to_utc, \ - convert_cocoa_core_data_ts_to_utc, check_in_embedded_media, artifact_processor, logfunc, is_platform_windows +from scripts.ilapfuncs import get_file_path, open_sqlite_db_readonly, get_sqlite_db_records, get_plist_content, get_plist_file_content, lava_get_full_media_info, \ + convert_unix_ts_to_utc, convert_cocoa_core_data_ts_to_utc, convert_ts_int_to_utc, check_in_media, check_in_embedded_media, artifact_processor, logfunc # -map_id_name = {} +bereal_user_map = {} # bereal user id bereal_user_id = None +# bereal app id +bereal_app_identifier = None +# bereal images +bereal_images = () +# constants +LINE_BREAK = '\n' +COMMA_SEP = ', ' +HTML_LINE_BREAK = '
' -def format_userid(id, name=None): - if id: - # "id (name)" - if name: - return f"{id} ({name})" - # "id (m_name)" - else: - m_name = map_id_name.get(id) - return f"{id} ({m_name})" if bool(m_name) else id - else: - return "Local User" - - -def get_json_data(file_path): +def get_json_file_content(file_path): try: with open(file_path, 'r', encoding='utf-8') as file: return json.load(file) @@ -255,16 +269,75 @@ def get_json_data(file_path): return None +def get_json_content(data): + try: + return json.loads(data) + except Exception as e: + logfunc(f"Unexpected error reading json data: {str(e)}") + return {} + + +def get_sqlite_db_records_regexpr(file_path, query, attach_query=None, regexpr=None): + file_path = str(file_path) + db = open_sqlite_db_readonly(file_path) + if not bool(db): + return None + + try: + # regexp() user function + if bool(regexpr): + db.create_function('regexp', 2, lambda x, y: 1 if re.search(x,y) else 0) + + cursor = db.cursor() + if bool(attach_query): + cursor.execute(attach_query) + cursor.execute(query) + records = cursor.fetchall() + return records + + except sqlite3.OperationalError as e: + logfunc(f"Error with {file_path}:") + logfunc(f" - {str(e)}") + + except sqlite3.ProgrammingError as e: + logfunc(f"Error with {file_path}:") + logfunc(f" - {str(e)}") + + +# Cache.db query +BEREAL_CACHE_DB_QUERY = ''' + SELECT + crd.entry_ID, + cr.request_key, + crd.isDataOnFS, + crd.receiver_data + FROM cfurl_cache_response AS "cr" + LEFT JOIN cfurl_cache_receiver_data AS "crd" ON (cr.entry_ID = crd.entry_ID) + WHERE cr.request_key REGEXP "{0}" + ''' + + +# iso 8601 format to utc +def convert_iso8601_to_utc(str_date): + if bool(str_date) and isinstance(str_date, str) and (str_date != 'null'): + if len(str_date) <= 10: + str_date = str_date + 'T00:00:00.000Z' + dt = datetime.fromisoformat(str_date).timestamp() + return convert_ts_int_to_utc(dt) + else: + return str_date + + # get key0 def get_key0(obj): return list(obj.keys())[0] if bool(obj) and isinstance(obj, dict) else None # profile picture or thumbnail or media or primary or secondary -def get_media(obj : dict): +def get_media(obj): if bool(obj) and isinstance(obj, dict): # url - pp_url = obj.get('url') + pp_url = obj.get('url', '') # size 1.x if bool(pp_size := obj.get('size')) and len(pp_size) == 2: pp_size = f"{pp_size[0]}x{pp_size[1]}" @@ -286,7 +359,7 @@ def get_media(obj : dict): else: pp_mt = None else: - pp_url = None + pp_url = '' pp_size = None pp_mt = None @@ -294,24 +367,129 @@ def get_media(obj : dict): return pp_mt, pp_url, pp_size +# placeholder from primary or secondary or thumbnail +def get_place_holder0_url(obj): + result = None + if bool(obj): + place_holders = obj.get('placeholders', []) + if bool(place_holders): + result = place_holders[0].get('url') + + return result + + +# music url +def get_music_artist_and_track(obj): + result = None + + if bool(obj) and isinstance(obj, dict) and bool(music := obj.get('music')): + # artist + artist = music.get('artist') + # track + track = music.get('track') + # artist and track + if bool(track): + result = f"{artist} - {track}" if bool(artist) else track + else: + result = artist if bool(artist) else None + + return result + + +# music url +def get_music_url(obj, html_format=False): + song_url = None + + if bool(obj) and isinstance(obj, dict) and bool(music := obj.get('music')): + # openUrl? + song_url = music.get('openUrl') + if not bool(song_url): + # provider (spotify, apple) + provider = music.get('provider') + # provider id/track id + track_id = music.get('providerId') + # album id + album_id = None + # audio type (track, ...) + # audioType + # spotify + if provider == 'spotify' and bool(track_id): + # old link: "https://open.spotify.com/track/{track_id}"" + # new link: "https://open.spotify.com/intl-it/track/{track_id}" + song_url = f"https://open.spotify.com/track/{track_id}" + # apple + elif ((provider == 'apple') or (provider.lower == 'apple music')) and bool(track_id): + # link: "https://music.apple.com/us/album/{album_id}?i={track_id}" +# song_url = f"https://music.apple.com/us/album/{album_id}?i={track_id}" + song_url = '?' + + # html? + if html_format: + song_url = generic_url(song_url, html_format=html_format) + + return song_url + + # profile type (regular, brand, celebrity, ...) def get_profile_type(obj): + REGULAR = 'User' + REAL_BRAND = 'RealBrand' + REAL_PEOPLE = 'RealPeople' profile_type = None - # profile type 1.x - is_real_people = obj.get('isRealPeople') - if is_real_people is not None: - profile_type = 'regular' if (is_real_people == False) else 'real_people' - # profile type 2.x, 4.x + if isinstance(obj, str): + profile_type = obj else: - profile_type = get_key0(obj.get('profileType')) + # type + _type = obj.get('type', '') + # regular + if (_type == 'USER'): + profile_type = REGULAR + # real brand + elif (_type == 'REAL_BRAND'): + profile_type = REAL_BRAND + # real people/celebrity + elif (_type == 'REAL_PERSON'): + profile_type = REAL_PEOPLE + # no type/unknown type + else: + profile_type = _type + # profile type + if bool(profile_type) == False: + profile_type = get_key0(obj.get('profileType')) + # official account profile type + if bool(profile_type) == False: + profile_type = get_key0(obj.get('officialAccountProfileType')) + + # friendly name + profile_type_lower = profile_type.lower() if bool(profile_type) else profile_type + # regular + if (bool(profile_type_lower) == False) or (profile_type_lower == 'regular'): + profile_type = REGULAR + # real brand + elif profile_type_lower == 'brand': + profile_type = REAL_BRAND + # real people/celebrity + elif profile_type_lower == 'celebrity': + profile_type = REAL_PEOPLE return profile_type +# format post type +def format_post_type(is_late, is_main): + # post type + post_type = 'late' if is_late else 'onTime' + if not is_main: + post_type = f"{post_type} bonus" + + return post_type + + # post type (onTime, late, ...) def get_post_type(obj, is_late=False): post_type = obj.get('postType', '') + if post_type.lower() == 'bonus': post_type = f"late {post_type}" if is_late else f"onTime {post_type}" @@ -319,41 +497,73 @@ def get_post_type(obj, is_late=False): # post visibilities -def get_post_visibilities(obj): - visibilities = obj.get('visibilities') - if visibilities is not None: - return ', '.join(obj) +def get_post_visibilities(obj, html_format=False): + if isinstance(obj, dict): + # friends, friends-of-friends, public + visibilities = obj.get('visibilities') + if not visibilities: + visibilities = obj.get('visibility') + elif isinstance(obj, list): + visibilities = obj else: - return None + visibilities = None + + if visibilities: + return HTML_LINE_BREAK.join(visibilities) if html_format else LINE_BREAK.join(visibilities) + else: + return 'N/D' + + +# late secs to time +def get_late_secs(obj): + # late in seconds + if isinstance(obj, dict): + late_secs = obj.get('lateInSeconds') + elif isinstance(obj, int): + late_secs = obj + else: + late_secs = None + return str(timedelta(seconds=late_secs)) if bool(late_secs) else late_secs # generic url def generic_url(value, html_format=False): - if value != None and len(value) > 0: + # default + result = None + + if bool(value) and (value != 'null'): u = urlparse(value) # 0=scheme, 2=path - if not bool(u[0]) and u[2].startswith('www'): + if not bool(u.scheme) and u.path.startswith('www'): u = u._replace(scheme='http') url = urlunparse(u) - return value if not html_format else f'{value}' - else: + result = f'{value}' if html_format else url + + return result + + +# unordered list +def unordered_list(values, html_format=False): + if not bool(values): return None + return HTML_LINE_BREAK.join(values) if html_format else LINE_BREAK.join(values) + # links def get_links(obj, html_format=False): - if obj: + if bool(obj) and isinstance(obj, list): links_urls = [] # array for i in range(0, len(obj)): - url = generic_url(obj[i].get('url'), html_format) + url = obj[i].get('url') + if not bool(url): + url = obj[i].get('link') # json + url = generic_url(url, html_format=html_format) links_urls.append(url) - if html_format: - return '
'.join(links_urls) - else: - return '\n'.join(links_urls) + return unordered_list(links_urls, html_format=html_format) else: - return None + return '' # realmojis @@ -368,34 +578,40 @@ def get_realmojis(obj, html_format=False): # real moji real_moji = real_mojis[i] # emoji->url - url_moji = generic_url(real_moji.get('media', {}).get('url'), html_format) + url_moji = generic_url(real_moji.get('media', {}).get('url'), html_format=html_format) all_mojis.append(f"{real_moji.get('emoji')} {url_moji}") - if html_format: - return '
'.join(all_mojis) - else: - return '\n'.join(all_mojis) + + return unordered_list(all_mojis, html_format=html_format) + # realMojis elif bool(real_mojis := obj.get('realMojis')): # array for i in range(0, len(real_mojis)): # real moji real_moji = real_mojis[i] - # moji identifier - #id = real_moji.get('id') - # uid - #uid = real_moji.get('uid') - # user name - #user_name = real_moji.get('userName') - # date - #reaction_date = convert_cocoa_core_data_ts_to_utc(real_moji.get('date')) # emoji->uri - uri_moji = generic_url(real_moji.get('uri'), html_format) + uri_moji = generic_url(real_moji.get('uri'), html_format=html_format) all_mojis.append(f"{real_moji.get('emoji')} {uri_moji}") - return '
'.join(all_mojis) if html_format else '\n'.join(all_mojis) + return unordered_list(all_mojis, html_format=html_format) + # none else: - return None + return '' + + +# string "id (user name|full name)" +def format_userid(id, name=None): + if id: + # "id (name)" + if name: + return f"{id} ({name})" + # "id (m_name)" + else: + m_name = bereal_user_map.get(id) + return f"{id} ({m_name})" if bool(m_name) else id + else: + return "Local User" # get_user @@ -419,7 +635,7 @@ def get_user(obj): # get_tags -def get_tags(obj, html_format=False): +def get_tags(obj, html_format=False, as_ul=False): tag_list = [] tags = obj.get('tags') @@ -431,27 +647,133 @@ def get_tags(obj, html_format=False): for i in range(0, len(tags)): tag = tags[i] - # V1? - if bool(tag.get('tagName')): - # userid (fullname) - user = format_userid(tag.get('userId')) - # V2 - else: - id = tag.get('id') - fullname = tag.get('fullname') + # has user? + user = tag.get('user') + if bool(user): + id = user.get('id') + fullname = user.get('fullname') # userid (fullname) if bool(fullname): user = f"{id} ({fullname})" - # userid (fullname|username) + # userid (username) else: - username = tag.get('username') + username = user.get('username') user = f"{id} ({username})" + else: + # V1? + id = tag.get('userId') + if bool(id): + # userid (fullname) + user = format_userid(id) + # V2 + else: + id = tag.get('id') + fullname = tag.get('fullname') + # userid (fullname) + if bool(fullname): + user = f"{id} ({fullname})" + # userid (username) + else: + username = tag.get('username') + user = f"{id} ({username})" if bool(user): tag_list.append(user) - - return '
'.join(tag_list) if html_format else '\n'.join(tag_list) + return unordered_list(tag_list, html_format=html_format) + + +# get_devices -> [ uid, info, app_ver, timezone ] +def get_devices(obj, html_format=False): + if bool(devices := obj.get('devices')): + # devices + dev_uid = [] + dev_info = [] + app_ver = [] + timezone = [] + # array + for i in range(0, len(devices)): + # device + device = devices[i] + # device uid + dev_uid.append(device.get('deviceId')) + # device info (model, ios ver) + dev_info.append(device.get('device')) + # app version + app_ver.append(device.get('clientVersion')) + # timezone + timezone.append(device.get('timezone')) + line_sep = HTML_LINE_BREAK if html_format else LINE_BREAK + return [ line_sep.join(dev_uid), line_sep.join(dev_info), line_sep.join(app_ver), line_sep.join(timezone) ] + else: + return None, None, None, None + + +# media item file from url +def media_item_from_url(seeker, url, artifact_info, from_photos=False): + file_path = None + + # https://cdn-us1.bereal.network/cdn-cgi/image/height=130/Photos/LRHP7nD4UhfzEJacyRtuZE37txxx/post/a2jRGMvcxwdfkNEI.webp + if bool(url): + i_photos = url.find('/Photos/') + if i_photos > -1: + # Photos/LRHP7nD4UhfzEJacyRtuZE37txxx/post/a2jRGMvcxwdfkNEI.webp + if from_photos: + url_path = url[i_photos + 1:] + # cdn-cgi/image/height=130/Photos/LRHP7nD4UhfzEJacyRtuZE37txxx/post/a2jRGMvcxwdfkNEI.webp + else: + url_path = str(urlparse(url).path[1:]) + else: + url_path = url + + # is thumbnail? yes -> /Library/Caches/com.hackemist.SDImageCache/memories_thumbnails_cache/ + if '/image/height=' in url: + img_dir = 'memories_thumbnails_cache' + # default /Library/Caches/com.hackemist.SDImageCache/default/ + else: + img_dir = 'default' + + hash = hashlib.md5(url_path.encode('utf-8')).hexdigest() + rel_path = Path(url_path).with_stem(hash) + file_name = rel_path.name + cache_pattern = str(Path(f"/{bereal_app_identifier}/Library/Caches/com.hackemist.SDImageCache/{img_dir}/{file_name}")) + for image in bereal_images: + if image.endswith(cache_pattern): + file_path = check_in_media(seeker, image, artifact_info, already_extracted=bereal_images) + break + + return file_path + + +# device path/local path +def get_device_file_path(file_path, seeker): + device_path = file_path + + if bool(file_path): + file_info = seeker.file_infos.get(file_path) if file_path else None + # data folder: /path/to/report/data + if file_info: + source_path = file_info.source_path + # extraction folder: /path/to/directory + else: + source_path = file_path + source_path = Path(source_path).as_posix() + + index_private = source_path.find('/private/') + if index_private > 0: + device_path = source_path[index_private:] + + return device_path + + +def get_cache_db_fs_path(data, file_found, seeker): + if bool(data): + # *//Library/Caches/AlexisBarreyat.BeReal/fsCachedData/ + filter = Path('*').joinpath(*Path(file_found).parts[-5:-1], 'fsCachedData', data) + json_file = seeker.search(filter, return_on_first_hit=True) + return json_file + else: + return None # preferences @@ -460,9 +782,13 @@ def bereal_preferences(files_found, report_folder, seeker, wrap_text, timezone_o source_path = None global bereal_user_id + global bereal_app_identifier + global bereal_images + artifact_info = inspect.stack()[0] # all files for file_found in files_found: + file_found = str(file_found) # prefs plist_data = get_plist_file_content(file_found) if not bool(plist_data): @@ -473,7 +799,7 @@ def bereal_preferences(files_found, report_folder, seeker, wrap_text, timezone_o source_path = file_found # group - if str(file_found).endswith('group.BeReal.plist'): + if file_found.endswith('group.BeReal.plist'): # me user_name = 'Local User' user_id = plist_data.get('bereal-user-id') @@ -484,20 +810,20 @@ def bereal_preferences(files_found, report_folder, seeker, wrap_text, timezone_o if bool(user_id): user_name = plist_data.get('myAccount', {}).get(user_id, {}).get('username') if not bool(user_name): user_name = 'Local User' - map_id_name[user_id] = user_name + bereal_user_map[user_id] = user_name # bereal user id bereal_user_id = user_id - # local profile picture (file:///private/var/mobile/Containers/Shared/AppGroup//notification/file.jpg) - bereal_profile_picture = plist_data.get('myAccount', {}).get(user_id, {}).get('profilePictureURL') # current friends current_friends = plist_data.get('currentFriends', {}) for user_id, user_name in current_friends.items(): - map_id_name[user_id] = user_name - - # app - else: - continue + bereal_user_map[user_id] = user_name + + # preferences + elif file_found.endswith('AlexisBarreyat.BeReal.plist'): + bereal_app_identifier = Path(file_found).parents[2].name + cache_pattern = str(Path('*', bereal_app_identifier, 'Library', 'Caches', 'com.hackemist.SDImageCache', '**', '*')) + bereal_images = seeker.search(cache_pattern, return_on_first_hit=False) except Exception as e: logfunc(f"Error: {str(e)}") @@ -511,162 +837,306 @@ def bereal_preferences(files_found, report_folder, seeker, wrap_text, timezone_o @artifact_processor def bereal_accounts(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Created', 'Profile type', 'Full name', 'User name', 'Profile picture URL', 'Gender', 'Birthday', 'Biography', - 'Country code', 'Region', 'Address', 'Timezone', 'Phone number', 'Device UID', 'Device Model and OS', 'App version', - 'Private', 'RealMojis', 'User ID', 'Source file name' ] + data_headers = ( + ('Created', 'datetime'), + 'Profile type', + 'Full name', + 'User name', + 'Profile picture URL', + ('Profile picture', 'media', 'height: 96px; border-radius: 50%;'), + 'Gender', + ('Birthday', 'date'), + 'Biography', + 'Country code', + 'Region', + 'Address', + 'Timezone', + ('Phone number', 'phonenumber'), + 'Device UID', + 'Device Model and OS', + 'App version', + 'Private', + 'RealMojis', + 'User ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_accounts']['name'] # all files for file_found in files_found: - # accounts - json_data = get_json_data(file_found) - if not bool(json_data): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + # disk-bereal-ProfileRepository + if file_rel_path.startswith('disk-bereal-ProfileRepository'): + try: + device_file_paths = [ device_file_path ] - # account - account = json_data.get('object') - if not bool(account): - continue + # json data + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - # created - created = convert_cocoa_core_data_ts_to_utc(account.get('createdAt')) - # profile type - profile_type = get_profile_type(account) - # full name - fullname = account.get('fullname') - # username - username = account.get('username') - # profile picture url - pp_mt, pp_url, pp_size = get_media(account.get('profilePicture')) - pp_url_html = generic_url(pp_url, html_format = True) - # gender - gender = get_key0(account.get('gender')) - # birth date - birth_date = convert_cocoa_core_data_ts_to_utc(account.get('birthDate')) - # biography - biography = account.get('biography') - # country code - country_code = account.get('countryCode') - # region - region = account.get('region') - # 2.x, 4.x - if bool(region) and isinstance(region, dict): - region = account.get('region', {}).get('value') - # location/address - address = account.get('location') - # phone number - phone_number = account.get('phoneNumber') - # devices - dev_uid = [] - dev_info = [] - app_ver = [] - timezone = [] - dev_uid_html = [] - dev_info_html = [] - app_ver_html = [] - timezone_html = [] - devices = account.get('devices') - if bool(devices): - # array - for i in range(0, len(devices)): - # device - device = devices[i] - # device uid - dev_uid.append(device.get('deviceId')) - dev_uid_html = dev_uid - # device info (model, ios ver) - dev_info.append(device.get('device')) - dev_info_html = dev_info - # app version - app_ver.append(device.get('clientVersion')) - app_ver_html = app_ver - # timezone - timezone.append(device.get('timezone')) - timezone_html = timezone - dev_uid = '\n'.join(dev_uid) - dev_info = '\n'.join(dev_info) - app_ver = '\n'.join(app_ver) - timezone = '\n'.join(timezone) - dev_uid_html = '
'.join(dev_uid_html) - dev_info_html = '
'.join(dev_info_html) - app_ver_html = '
'.join(app_ver_html) - timezone_html = '
'.join(timezone_html) - # is private? - is_private = account.get('isPrivate') - # realmojis - realmojis = get_realmojis(account, html_format = False) - realmojis_html = get_realmojis(account, html_format = True) - # unique id/user id - id = account.get('id') - - # html row - data_list_html.append((created, profile_type, fullname, username, pp_url_html, gender, birth_date, biography, - country_code, region, address, timezone_html, phone_number, dev_uid_html, dev_info_html, app_ver_html, - is_private, realmojis_html, id, file_rel_path)) - - # lava row - data_list.append((created, profile_type, fullname, username, pp_url, gender, birth_date, biography, - country_code, region, address, timezone, phone_number, dev_uid, dev_info, app_ver, - is_private, realmojis, id, file_rel_path)) + # account + account = json_data.get('object') + if not bool(account): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass + # created + created = convert_cocoa_core_data_ts_to_utc(account.get('createdAt')) + # profile type + profile_type = get_profile_type(account) + # full name + fullname = account.get('fullname') + # username + username = account.get('username') + # profile picture url + pp_mt, pp_url, pp_size = get_media(account.get('profilePicture')) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # gender + gender = get_key0(account.get('gender')) + # birth date + birth_date = convert_cocoa_core_data_ts_to_utc(account.get('birthDate')) + # biography + biography = account.get('biography') + # country code + country_code = account.get('countryCode') + # region + region = account.get('region') + # 2.x, 4.x + if bool(region) and isinstance(region, dict): + region = account.get('region', {}).get('value') + # location/address + address = account.get('location') + # phone number + phone_number = account.get('phoneNumber') + # devices + devices = get_devices(account) + devices_html = get_devices(account, html_format=True) + # is private? + is_private = account.get('isPrivate') + # realmojis + realmojis = get_realmojis(account) + realmojis_html = get_realmojis(account, html_format=True) + # unique id/user id + id = account.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object]" + + # html row + data_list_html.append((created, profile_type, fullname, username, pp_url_html, pp_media_ref_id, gender, birth_date, biography, + country_code, region, address, devices_html[3], phone_number, devices_html[0], devices_html[1], devices_html[2], + is_private, realmojis_html, id, source_file_name_html, location)) + # lava row + data_list.append((created, profile_type, fullname, username, pp_url, pp_media_ref_id, gender, birth_date, biography, + country_code, region, address, devices[3], phone_number, devices[0], devices[1], devices[2], + is_private, realmojis, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/person\/me$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) == 0: + continue + + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + device_file_paths = db_device_file_paths + + # accounts + if not (bool(json_data) or isinstance(json_data, dict)): + continue - # lava types - data_headers[0] = (data_headers[0], 'datetime') - data_headers[6] = (data_headers[6], 'date') - data_headers[12] = (data_headers[12], 'phonenumber') + # account + account = json_data + if not bool(account): + continue + + # created + created = convert_iso8601_to_utc(account.get('createdAt')) + # profile type + profile_type = get_profile_type(account) + # full name + fullname = account.get('fullname') + # username + username = account.get('username') + # profile picture url + pp_mt, pp_url, pp_size = get_media(account.get('profilePicture')) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # gender + gender = account.get('gender') + # birth date + birth_date = convert_iso8601_to_utc(account.get('birthdate')) + # biography + biography = account.get('biography') + # country code + country_code = account.get('countryCode') + # region + region = account.get('region') + # location/address + address = account.get('location') + # phone number + phone_number = account.get('phoneNumber') + # devices + devices = get_devices(account) + devices_html = get_devices(account, html_format=True) + # is private? + is_private = account.get('isPrivate') + # realmojis + realmojis = get_realmojis(account) + realmojis_html = get_realmojis(account, html_format=True) + # unique id/user id + id = account.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((created, profile_type, fullname, username, pp_url_html, pp_media_ref_id, gender, birth_date, biography, + country_code, region, address, devices_html[3], phone_number, devices_html[0], devices_html[1], devices_html[2], + is_private, realmojis_html, id, source_file_name_html, location)) + # lava row + data_list.append((created, profile_type, fullname, username, pp_url, pp_media_ref_id, gender, birth_date, biography, + country_code, region, address, devices[3], phone_number, devices[0], devices[1], devices[2], + is_private, realmojis, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Accounts: " + str(ex)) + pass + + # EntitiesStore.sqlite + elif file_rel_path.endswith('EntitiesStore.sqlite'): + try: + query = ''' + SELECT + U.Z_PK, + U.ZPROFILETYPE, + U.ZFULLNAME, + U.ZUSERNAME, + U.ZPROFILEPICTUREURL, + U.ZBIO, + U.ZID + FROM ZUSERMO AS "U" + WHERE U.ZID = "{0}" + '''.format(bereal_user_id) + db_records = get_sqlite_db_records(file_found, query) + if len(db_records) == 0: + continue + + for record in db_records: + device_file_paths = [ device_file_path ] + + # profile type + profile_type = get_profile_type(record[1]) + # full name + fullname = record[2] + # username + username = record[3] + # profile picture url + pp_url = record[4] + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # biography + biography = record[5] + # unique id/user id + id = record[6] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"ZUSERMO (Z_PK: {record[0]})" ] + location = COMMA_SEP.join(location) - # paths - source_path = ', '.join(source_paths) + # html row + data_list_html.append((None, profile_type, fullname, username, pp_url_html, pp_media_ref_id, None, None, biography, + None, None, None, None, None, None, None, None, + None, None, id, source_file_name_html, location)) + # lava row + data_list.append((None, profile_type, fullname, username, pp_url, pp_media_ref_id, None, None, biography, + None, None, None, None, None, None, None, None, + None, None, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - EntitiesStore.sqlite Accounts: " + str(ex)) + pass - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' # contacts @artifact_processor def bereal_contacts(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = ('Full name', - 'Family name', - 'Middle name', - 'Given name', 'Nick name', - ('Profile picture', 'media', 'height: 80px'), - 'Organization name', - 'Phone numbers', - 'Source file name', - 'Location') + data_headers = ( + 'Full name', + 'Family name', + 'Middle name', + 'Given name', + 'Nick name', + ('Profile picture', 'media', 'height: 96px; border-radius: 50%;'), + 'Organization name', + 'Phone numbers', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_contacts']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + # json data + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue # contacts contacts = json_data.get('object') @@ -675,6 +1145,8 @@ def bereal_contacts(files_found, report_folder, seeker, wrap_text, timezone_offs # array for i in range(0, len(contacts)): + device_file_paths = [ device_file_path ] + contact = contacts[i] if not bool(contact): continue @@ -693,162 +1165,356 @@ def bereal_contacts(files_found, report_folder, seeker, wrap_text, timezone_offs photo_b64 = contact.get('photo') if bool(photo_b64): photo_raw = standard_b64decode(photo_b64) # bytes - photo = check_in_embedded_media(seeker, file_found, photo_raw, artifact_info) + pp_media_ref_id = check_in_embedded_media(seeker, file_found, photo_raw, artifact_info) else: - photo = '' + pp_media_ref_id = None # organization name organization_name = contact.get('organizationName') # phone numbers - phone_numbers = '\n'.join(contact.get('phoneNumbers')) - phone_numbers_html = '
'.join(contact.get('phoneNumbers')) + phone_numbers = LINE_BREAK.join(contact.get('phoneNumbers')) + phone_numbers_html = HTML_LINE_BREAK.join(contact.get('phoneNumbers')) + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}]" # html row - data_list_html.append((full_name, family_name, middle_name, given_name, nick_name, photo, organization_name, - phone_numbers_html, file_rel_path, location)) - + data_list_html.append((full_name, family_name, middle_name, given_name, nick_name, pp_media_ref_id, organization_name, + phone_numbers_html, source_file_name_html, location)) # lava row - data_list.append((full_name, family_name, middle_name, given_name, nick_name, photo, organization_name, - phone_numbers, file_rel_path, location)) - - except Exception as e: - logfunc(f"Error: {str(e)}") + data_list.append((full_name, family_name, middle_name, given_name, nick_name, pp_media_ref_id, organization_name, + phone_numbers, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) pass - # paths - source_path = ', '.join(source_paths) - - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' # persons @artifact_processor def bereal_persons(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Created', 'Profile type', 'Full name', 'User name', 'Profile picture URL', 'Biography', 'Address', - 'Relationship', 'Friended at', 'Urls', 'Streak count', 'Unique ID', 'Source file name', 'Location' ] + data_headers = ( + ('Created', 'datetime'), + 'Profile type', + 'Full name', + 'User name', + 'Profile picture URL', + ('Profile picture', 'media', 'height: 96px; border-radius: 50%;'), + 'Biography', + 'Address', + 'Relationship', + ('Friended at', 'datetime'), + 'URLs', + 'Streak count', + 'Unique ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_persons']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) - - # person - person = json_data.get('object') - if not bool(person): - continue + # PersonRepository + if file_rel_path.startswith('PersonRepository'): + try: + device_file_paths = [ device_file_path ] - # created (utc) - created = person.get('createdAt') - created = convert_cocoa_core_data_ts_to_utc(created) - # profile type - profile_type = get_profile_type(person) - # fullname - fullname = person.get('fullname') - # username - username = person.get('username') - # profile picture url - pp_url = person.get('profilePictureURL') - pp_url_html = generic_url(pp_url, html_format = True) - # biography - biography = person.get('biography') - # location - address = person.get('location') - # relationship - relationship = person.get('relationship') - if bool(relationship): - relationship_type = '' - # status - relationship_status = relationship.get('status') - if relationship_status == 'accepted' or relationship_status == 'pending': - relationship_type = 'Friend' - # friended at - relationship_friended_at = convert_cocoa_core_data_ts_to_utc(relationship.get('friendedAt')) - else: - relationship_type = '' # Following? - relationship_friended_at = '' - # links - links = get_links(person.get('links'), html_format = False) - links_html = get_links(person.get('links'), html_format = True) - # streak count - streak_count = person.get('streakCount') - # unique id - id = person.get('id') - - # location - location = f"[object]" - - # html row - data_list_html.append((created, profile_type, fullname, username, pp_url_html, biography, address, - relationship_type, relationship_friended_at, links_html, streak_count, id, file_rel_path, location)) - - # lava row - data_list.append((created, profile_type, fullname, username, pp_url, biography, address, - relationship_type, relationship_friended_at, links, streak_count, id, file_rel_path, location)) + # json data + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass - - # lava types - data_headers[0] = (data_headers[0], 'datetime') - data_headers[8] = (data_headers[8], 'datetime') + # person + person = json_data.get('object') + if not bool(person): + continue + + # created (utc) + created = convert_cocoa_core_data_ts_to_utc(person.get('createdAt')) + # profile type + profile_type = get_profile_type(person) + # fullname + fullname = person.get('fullname') + # username + username = person.get('username') + # profile picture url + pp_url = person.get('profilePictureURL') + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # biography + biography = person.get('biography') + # location + address = person.get('location') + # relationship + relationship = person.get('relationship') + if bool(relationship): + relationship_type = '' + # status + relationship_status = relationship.get('status') + if relationship_status == 'accepted' or relationship_status == 'pending': + relationship_type = 'Friend' + # friended at + relationship_friended_at = convert_cocoa_core_data_ts_to_utc(relationship.get('friendedAt')) + else: + relationship_type = '' # Following? + relationship_friended_at = '' + # links + links = get_links(person.get('links')) + links_html = get_links(person.get('links'), html_format=True) + # streak count + streak_count = person.get('streakCount') + # unique id + id = person.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object]" + + # html row + data_list_html.append((created, profile_type, fullname, username, pp_url_html, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links_html, streak_count, id, source_file_name_html, location)) + # lava row + data_list.append((created, profile_type, fullname, username, pp_url, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links, streak_count, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-Production_officialAccountProfiles + elif file_rel_path.startswith('disk-bereal-Production_officialAccountProfiles'): + try: + # json data + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # persons + persons = json_data.get('object') + if not bool(persons): + continue + for uid, person in persons.items(): + device_file_paths = [ device_file_path ] + + # uids + if not bool(person): + continue + + # created (utc) + created = None + # profile type + profile_type = get_profile_type(person) + # fullname + fullname = person.get('fullname') + # username + username = person.get('username') + # profile picture url + pp_url = person.get('profilePictureURL') + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # biography + biography = person.get('biography') + # location + address = person.get('location') + # relationship + relationship_type = '' # Following? + relationship_friended_at = '' + # links + links = get_links(person.get('links')) + links_html = get_links(person.get('links'), html_format=True) + # streak count + streak_count = None + # unique id + id = person.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][{uid}]" + + # html row + data_list_html.append((created, profile_type, fullname, username, pp_url_html, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links_html, streak_count, id, source_file_name_html, location)) + # lava row + data_list.append((created, profile_type, fullname, username, pp_url, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links, streak_count, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/person\/profiles\/[\w-]+\?') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) == 0: + continue + + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + device_file_paths = db_device_file_paths + + # person + if not (bool(json_data) or isinstance(json_data, dict)): + continue - # paths - source_path = ', '.join(source_paths) + # person + person = json_data + if not bool(person): + continue + + # created (utc) + created = convert_iso8601_to_utc(person.get('createdAt')) + # profile type + profile_type = get_profile_type(person) + # fullname + fullname = person.get('fullname') + # username + username = person.get('username') + # profile picture url + pp_mt, pp_url, pp_size = get_media(person.get('profilePicture')) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # biography + biography = person.get('biography') + # location + address = person.get('location') + # relationship + relationship = person.get('relationship') + if bool(relationship): + relationship_type = '' + # status + relationship_status = relationship.get('status') + if relationship_status == 'accepted' or relationship_status == 'pending': + relationship_type = 'Friend' + # friended at + relationship_friended_at = convert_iso8601_to_utc(relationship.get('friendedAt')) + else: + relationship_type = None # Following? + relationship_friended_at = None + # links + links = get_links(person.get('links')) + links_html = get_links(person.get('links'), html_format=True) + # streak length + streak_count = person.get('streakLength') + # unique id + id = person.get('id') - return data_headers, (data_list, data_list_html), source_path + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((created, profile_type, fullname, username, pp_url_html, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links_html, streak_count, id, source_file_name_html, location)) + # lava row + data_list.append((created, profile_type, fullname, username, pp_url, pp_media_ref_id, biography, address, + relationship_type, relationship_friended_at, links, streak_count, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Persons: " + str(ex)) + pass + + return data_headers, (data_list, data_list_html), ' ' # friends @artifact_processor def bereal_friends(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Status updated at', 'Status', 'Profile type', 'Full name', 'Username', 'Profile picture URL', 'Mutual friends', 'Unique ID', - 'Source file name', 'Location' ] + data_headers = ( + ('Status updated at', 'datetime'), + 'Status', + 'Profile type', + 'Full name', + 'Username', + 'Profile picture URL', + ('Profile picture', 'media', 'height: 96px; border-radius: 50%;'), + 'Mutual friends', + 'Unique ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_friends']['name'] + + # status + def status_friendly_name(status): + if bool(status): + value = status.lower() + if value == 'accepted': value = 'Friend' + elif value == 'sent': value = 'Request Sent' + elif value == 'pending': value = 'Request Received' + elif value == 'canceled': value = 'Request Canceled' + elif value == 'rejected': value = 'Request Rejected' + return value + else: + return '' + # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue - - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) - # object? - obj_ref = json_data.get('object') - if not bool(obj_ref): - continue + # following (dictionary) + # disk-bereal-Production_FriendsStorage.following/.following + # disk-bereal-Production_FriendsStorage.followers/.followers + if file_rel_path.endswith('.following') or file_rel_path.endswith('.followers'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue - # following (dictionary) - # disk-bereal-Production_FriendsStorage.following/.following - # disk-bereal-Production_FriendsStorage.followers/.followers - if file_rel_path.endswith('.following') or file_rel_path.endswith('.followers'): # relationship relationship = obj_ref.get('relationship', '') # users @@ -858,6 +1524,8 @@ def bereal_friends(files_found, report_folder, seeker, wrap_text, timezone_offse # array for i in range(0, len(obj_ref)): + device_file_paths = [ device_file_path ] + user = obj_ref[i] if not bool(user): continue @@ -872,29 +1540,53 @@ def bereal_friends(files_found, report_folder, seeker, wrap_text, timezone_offse username = user.get('username') # profile picture url pp_url = user.get('profilePictureURL') - pp_url_html = generic_url(pp_url, html_format = True) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) # unique id id = user.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][users][{i}]" # html row - data_list_html.append((None, status, profile_type, fullname, username, pp_url_html, None, id, - file_rel_path, location)) + data_list_html.append((None, status, profile_type, fullname, username, pp_url_html, pp_media_ref_id, + None, id, source_file_name_html, location)) # lava row - data_list.append((None, status, profile_type, fullname, username, pp_url, None, id, - file_rel_path, location)) - - # friends (array) - # disk-bereal-RelationshipsFriendsListManager/* - # disk-bereal-RelationshipsRequestSentListManager/* - # disk-bereal-RelationshipsRequestReceivedListManager/* - elif file_rel_path.startswith('disk-bereal-RelationshipsFriendsListManager') or \ - file_rel_path.startswith('disk-bereal-RelationshipsRequestSentListManager') or \ - file_rel_path.startswith('disk-bereal-RelationshipsRequestReceivedListManager'): + data_list.append((None, status, profile_type, fullname, username, pp_url, pp_media_ref_id, + None, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # friends (array) + # disk-bereal-RelationshipsFriendsListManager/* + # disk-bereal-RelationshipsRequestSentListManager/* + # disk-bereal-RelationshipsRequestReceivedListManager/* + elif file_rel_path.startswith('disk-bereal-RelationshipsFriendsListManager') or \ + file_rel_path.startswith('disk-bereal-RelationshipsRequestSentListManager') or \ + file_rel_path.startswith('disk-bereal-RelationshipsRequestReceivedListManager'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue + # array for i in range(0, len(obj_ref)): + device_file_paths = [ device_file_path ] + friend = obj_ref[i] if not friend: continue @@ -902,12 +1594,7 @@ def bereal_friends(files_found, report_folder, seeker, wrap_text, timezone_offse # status updated at status_updated_at = convert_cocoa_core_data_ts_to_utc(friend.get('statusUpdatedAt')) # status - status = friend.get('status', '').lower() - if status == 'accepted': status = 'Friend' - elif status == 'sent': status = 'Request Sent' - elif status == 'pending': status = 'Request Received' - elif status == 'canceled': status = 'Request Canceled' - elif status == 'rejected': status = 'Request Rejected' + status = status_friendly_name(friend.get('status')) # profile type profile_type = get_profile_type(friend) # fullname @@ -916,144 +1603,434 @@ def bereal_friends(files_found, report_folder, seeker, wrap_text, timezone_offse username = friend.get('username') # profile picture url pp_url = friend.get('profilePictureURL') - pp_url_html = generic_url(pp_url, html_format = True) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) # mutual friends mutual_friends = friend.get('mutualFriends') # unique id id = friend.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}]" # html row - data_list_html.append((status_updated_at, status, profile_type, fullname, username, pp_url_html, mutual_friends, id, - file_rel_path, location)) + data_list_html.append((status_updated_at, status, profile_type, fullname, username, pp_url_html, pp_media_ref_id, + mutual_friends, id, source_file_name_html, location)) # lava row - data_list.append((status_updated_at, status, profile_type, fullname, username, pp_url, mutual_friends, id, - file_rel_path, location)) + data_list.append((status_updated_at, status, profile_type, fullname, username, pp_url, pp_media_ref_id, + mutual_friends, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # friends + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/relationships\/friends($|\/\?page)') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + obj_ref = json_data.get('data') + if not bool(obj_ref): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass + # array + for i in range(0, len(obj_ref)): + device_file_paths = db_device_file_paths - # lava types - data_headers[0] = (data_headers[0], 'datetime') + friend = obj_ref[i] + if not friend: + continue - # paths - source_path = ', '.join(source_paths) + # status + status = status_friendly_name(friend.get('status')) + # profile type + profile_type = get_profile_type(friend) + # fullname + fullname = friend.get('fullname') + # username + username = friend.get('username') + # profile picture url + pp_mt, pp_url, pp_size = get_media(friend.get('profilePicture')) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # unique id + id = friend.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[data][{i}]") + location = COMMA_SEP.join(location) - return data_headers, (data_list, data_list_html), source_path + # html row + data_list_html.append((None, status, profile_type, fullname, username, pp_url_html, pp_media_ref_id, + None, id, source_file_name_html, location)) + # lava row + data_list.append((None, status, profile_type, fullname, username, pp_url, pp_media_ref_id, + None, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Friends: " + str(ex)) + pass + + try: + # friend requests (received/sent) + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/relationships\/friend-requests\/(received|sent)\?page') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + obj_ref = json_data.get('data') + if not bool(obj_ref): + continue + # array + for i in range(0, len(obj_ref)): + device_file_paths = db_device_file_paths -# blocked users -@artifact_processor -def bereal_blocked_users(files_found, report_folder, seeker, wrap_text, timezone_offset): + friend = obj_ref[i] + if not friend: + continue - data_headers = [ 'Blocked', 'Full name', 'Username', 'Profile picture URL', 'Unique ID', 'Source file name', 'Location' ] - data_list = [] - data_list_html = [] - source_paths = set() + # status updated at + status_updated_at = convert_iso8601_to_utc(friend.get('updatedAt')) + # status + status = status_friendly_name(friend.get('status')) + # profile type + profile_type = get_profile_type(friend) + # fullname + fullname = friend.get('fullname') + # username + username = friend.get('username') + # profile picture url + pp_mt, pp_url, pp_size = get_media(friend.get('profilePicture')) + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # mutual friends + mutual_friends = friend.get('mutualFriends') + # unique id + id = friend.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[data][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((status_updated_at, status, profile_type, fullname, username, pp_url_html, pp_media_ref_id, + mutual_friends, id, source_file_name_html, location)) + # lava row + data_list.append((status_updated_at, status, profile_type, fullname, username, pp_url, pp_media_ref_id, + mutual_friends, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Friend requests: " + str(ex)) + pass + + # EntitiesStore.sqlite + elif file_rel_path.endswith('EntitiesStore.sqlite'): + try: + query = ''' + SELECT + U.Z_PK, + U.ZPROFILETYPE, + U.ZFULLNAME, + U.ZUSERNAME, + U.ZPROFILEPICTUREURL, + U.ZBIO, + U.ZID + FROM ZUSERMO AS "U" + WHERE U.ZID != "{0}" + '''.format(bereal_user_id) + db_records = get_sqlite_db_records(file_found, query) + if len(db_records) == 0: + continue + + for record in db_records: + device_file_paths = [ device_file_path ] + + # profile type + profile_type = get_profile_type(record[1]) + # full name + fullname = record[2] + # username + username = record[3] + # profile picture url + pp_url = record[4] + pp_url_html = generic_url(pp_url, html_format=True) + # profile picture + pp_media_ref_id = media_item_from_url(seeker, pp_url, artifact_info) + pp_media_item = lava_get_full_media_info(pp_media_ref_id) + if pp_media_item: device_file_paths.append(get_device_file_path(pp_media_item[5], seeker)) + # biography + biography = record[5] + # unique id/user id + id = record[6] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"ZUSERMO (Z_PK: {record[0]})" ] + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((None, None, profile_type, fullname, username, pp_url_html, pp_media_ref_id, + None, id, source_file_name_html, location)) + # lava row + data_list.append((None, None, profile_type, fullname, username, pp_url, pp_media_ref_id, + None, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + return data_headers, (data_list, data_list_html), ' ' + + +# blocked users +@artifact_processor +def bereal_blocked_users(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Blocked', 'datetime'), + 'Full name', + 'Username', + 'Unique ID', + 'Source file name', + 'Location' + ) + data_list = [] + data_list_html = [] + device_file_paths = [] + artifact_info_name = __artifacts_v2__['bereal_blocked_users']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) + + # disk-bereal-BlockedUserManager + if file_rel_path.endswith('disk-bereal-BlockedUserManager'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + # users + users = json_data.get('object') + if not bool(users): + continue - # friends - friends = json_data.get('object') - if not bool(friends): - continue + # array + for i in range(0, len(users)): + device_file_paths = [ device_file_path ] - # array - for i in range(0, len(friends)): - friend = friends[i] - if not bool(friend): - continue + user = users[i] + if not bool(user): + continue - # blocked date - blocked_date = convert_cocoa_core_data_ts_to_utc(friend.get('blockedDate')) - # fullname - fullname = friend.get('fullname') - # username - username = friend.get('username') - # profile picture url - pp_url = friend.get('profilePictureURL') - pp_url_html = generic_url(pp_url, html_format = True) - # unique id - id = friend.get('id') + # blocked date + blocked_date = convert_cocoa_core_data_ts_to_utc(user.get('blockedDate')) + # fullname + fullname = user.get('fullname') + # username + username = user.get('username') + # unique id + id = user.get('id') - # location - location = f"[object][{i}]" + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][{i}]" - # html row - data_list_html.append((blocked_date, fullname, username, pp_url_html, id, file_rel_path, location)) + # html row + data_list_html.append((blocked_date, fullname, username, id, source_file_name_html, location)) + # lava row + data_list.append((blocked_date, fullname, username, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # blocked users + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/moderation\/block-users\?page') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) == 0: + continue - # lava row - data_list.append((blocked_date, fullname, username, pp_url, id, file_rel_path, location)) + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + users = json_data.get('data') + if not bool(users): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass + # array + for i in range(0, len(users)): + device_file_paths = db_device_file_paths - # lava types - data_headers[0] = (data_headers[0], 'datetime') + user = users[i] + if not user: + continue - # paths - source_path = ', '.join(source_paths) + # blocked date + blocked_date = convert_iso8601_to_utc(user.get('blockedAt')) + # fullname + fullname = user.get('user', {}).get('fullname') + # username + username = user.get('user', {}).get('username') + # unique id + id = user.get('user', {}).get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[data][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((blocked_date, fullname, username, id, source_file_name_html, location)) + # lava row + data_list.append((blocked_date, fullname, username, id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Blocked Users: " + str(ex)) + pass - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' # posts @artifact_processor def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Taken at', 'Moment day', 'Post type', 'Author', 'Primary media type', 'Primary URL', 'Secondary media type', 'Secondary URL', - 'Thumbnail media type', 'Thumbnail URL', 'Caption', 'Latitude', 'Longitude', 'Retake counter', 'Late time', 'Tagged friends', - 'Moment ID', 'BeReal ID', 'Source file name', 'Location' ] + data_headers = ( + ('Taken at', 'datetime'), + 'Post type', + 'Author', + 'Primary media type', + 'Primary URL', + ('Primary image', 'media', 'height: 96px; border-radius: 5%;'), + 'Secondary media type', + 'Secondary URL', + ('Secondary image', 'media', 'height: 96px; border-radius: 5%;'), + 'Thumbnail media type', + 'Thumbnail URL', + ('Thumbnail image', 'media', 'height: 96px; border-radius: 5%;'), + 'Song title', + 'Song URL', + 'Caption', + 'Latitude', + 'Longitude', + 'Retake counter', + 'Late time', + 'Visibility', + 'Tagged friends', + 'Moment ID', + 'BeReal ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_posts']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue - - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) + + # PersonRepository + if file_rel_path.startswith('PersonRepository'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - # object? - obj_ref = json_data.get('object') - if not bool(obj_ref): - continue + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue - # PersonRepository - if file_rel_path.startswith('PersonRepository'): # posts - posts = json_data.get('object', {}).get('beRealOfTheDay', {}).get('series', {}).get('posts') + posts = obj_ref.get('beRealOfTheDay', {}).get('series', {}).get('posts') if not bool(posts): continue # array for i in range(0, len(posts)): + device_file_paths = [ device_file_path ] + post = posts[i] if not bool(post): continue @@ -1062,18 +2039,12 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) bereal_id = post.get('id') # moment id moment_id = post.get('momentID') - # moment at - moment_at = None # is late? is_late = post.get('isLate') # late in seconds - late_secs = post.get('lateInSeconds') - if bool(late_secs): late_secs = str(timedelta(seconds=late_secs)) - # is main? - is_main = post.get('isMain') + late_secs = get_late_secs(post) # post type - post_type = 'late' if is_late else 'onTime' - if not is_main: post_type = f"{post_type} bonus" + post_type = format_post_type(is_late, post.get('isMain')) # caption caption = post.get('caption') # latitude @@ -1083,36 +2054,65 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) # author author_id, author_user_name = get_user(post) author = format_userid(author_id, author_user_name) - # post visibilities - post_visibilities = get_post_visibilities(post) + # visibilities + visibilities = get_post_visibilities(post) + visibilities_html = get_post_visibilities(post, html_format=True) # taken at taken_at = convert_cocoa_core_data_ts_to_utc(post.get('takenAt')) # retake counter retake_counter = post.get('retakeCounter') # primary p_mt, p_url, p_size = get_media(post.get('primaryMedia')) - p_url_html = generic_url(p_url, html_format = True) + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) # secondary s_mt, s_url, s_size = get_media(post.get('secondaryMedia')) - s_url_html = generic_url(s_url, html_format = True) + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # music + song_title = None + song_url = None + song_url_html = None # tags - tags = get_tags(post, html_format = False) - tags_html = get_tags(post, html_format = True) + tags = get_tags(post) + tags_html = get_tags(post, html_format=True) + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][beRealOfTheDay][series][posts][{i}]" # html row - data_list_html.append((taken_at, moment_at, post_type, author, p_mt, p_url_html, s_mt, s_url_html, - None, None, caption, latitude, longitude, retake_counter, late_secs, tags_html, - moment_id, bereal_id, file_rel_path, location)) + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + None, None, None, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) # lava row - data_list.append((taken_at, moment_at, post_type, author, p_mt, p_url, s_mt, s_url, - None, None, caption, latitude, longitude, retake_counter, late_secs, tags, - moment_id, bereal_id, file_rel_path, location)) - - # disk-bereal-MemoriesRepository-subject-key - elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + None, None, None, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-MemoriesRepository-subject-key + elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue + # author author = format_userid(bereal_user_id) @@ -1130,6 +2130,8 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) continue for j in range(0, len(detailed_posts)): + device_file_paths = [ device_file_path ] + # detailed post detailed_post = detailed_posts[j] if not bool(detailed_post): @@ -1139,17 +2141,14 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) bereal_id = detailed_post.get('id') # moment id moment_id = detailed_post.get('momentID') - # moment at - moment_at = convert_cocoa_core_data_ts_to_utc(detailed_post.get('momentAt')) # post type post_type = get_post_type(detailed_post, is_late) -# # late in seconds??? - late_secs = detailed_post.get('lateInSeconds') - if bool(late_secs): late_secs = str(timedelta(seconds=late_secs)) + # late in seconds??? + late_secs = get_late_secs(detailed_post) # metadata metadata = detailed_post.get('details', {}).get('full', {}).get('metadata') if not bool(metadata): - continue + continue # caption caption = metadata.get('caption') # latitude @@ -1157,47 +2156,85 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) # longitude longitude = metadata.get('location', {}).get('longitude') # tags - tags = get_tags(metadata, html_format = False) - tags_html = get_tags(metadata, html_format = True) + tags = get_tags(metadata) + tags_html = get_tags(metadata, html_format=True) + # visibilities + visibilities = visibilities_html = None # full data data = detailed_post.get('details', {}).get('full', {}).get('data') # preview data if not bool(data): data = detailed_post.get('details', {}).get('preview', {}).get('data') if not bool(data): - continue + continue # taken at taken_at = convert_cocoa_core_data_ts_to_utc(data.get('takenAt')) # retake counter retake_counter = data.get('retakeCounter') -# # late in seconds??? - if not bool(late_secs): - late_secs = data.get('lateInSeconds') - if bool(late_secs): str(timedelta(seconds=late_secs)) + # late in seconds??? + late_secs = get_late_secs(data) + # is video? + is_video = get_key0(data.get('postType')) == 'video' # primary - p_mt, p_url, p_size = get_media(data.get('primary')) - p_url_html = generic_url(p_url, html_format = True) + primary = data.get('primary', {}) + p_mt, p_url, p_size = get_media(primary) + p_url_html = generic_url(p_url, html_format=True) + if is_video: + p_media_ref_id = media_item_from_url(seeker, get_place_holder0_url(primary), artifact_info, from_photos=True) + else: + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) # secondary - s_mt, s_url, s_size = get_media(data.get('secondary')) - s_url_html = generic_url(s_url, html_format = True) + secondary = data.get('secondary', {}) + s_mt, s_url, s_size = get_media(secondary) + s_url_html = generic_url(s_url, html_format=True) + if is_video: + s_media_ref_id = media_item_from_url(seeker, get_place_holder0_url(secondary), artifact_info, from_photos=True) + else: + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) # thumbnail - t_mt, t_url, t_size = get_media(data.get('thumbnail')) - t_url_html = generic_url(t_url, html_format = True) - + thumbnail = data.get('thumbnail', {}) + t_mt, t_url, t_size = get_media(thumbnail) + t_url_html = generic_url(t_url, html_format=True) + t_media_ref_id = media_item_from_url(seeker, t_url, artifact_info, from_photos=is_video) + t_media_item = lava_get_full_media_info(t_media_ref_id) + if t_media_item: device_file_paths.append(get_device_file_path(t_media_item[5], seeker)) + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}]" # html row - data_list_html.append((taken_at, moment_at, post_type, author, p_mt, p_url_html, s_mt, s_url_html, - t_mt, t_url_html, caption, latitude, longitude, retake_counter, late_secs, tags_html, - moment_id, bereal_id, file_rel_path, location)) + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + t_mt, t_url_html, t_media_ref_id, None, None, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) # lava row - data_list.append((taken_at, moment_at, post_type, author, p_mt, p_url, s_mt, s_url, - t_mt, t_url, caption, latitude, longitude, retake_counter, late_secs, tags, - moment_id, bereal_id, file_rel_path, location)) - - # disk-bereal-Production_postFeedItems - elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + t_mt, t_url, t_media_ref_id, None, None, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-Production_postFeedItems + elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue + # array (string, array) for i in range(0, len(obj_ref)): # array @@ -1220,6 +2257,8 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) continue for p in range(0, len(posts)): + device_file_paths = [ device_file_path ] + post = posts[p] if not bool(post): continue @@ -1228,18 +2267,12 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) bereal_id = post.get('id') # moment id moment_id = post.get('momentID') - # moment at - moment_at = None # is late? is_late = post.get('isLate') # late in seconds - late_secs = post.get('lateInSeconds') - if bool(late_secs): late_secs = str(timedelta(seconds=late_secs)) - # is main? - is_main = post.get('isMain') + late_secs = get_late_secs(post) # post type - post_type = 'late' if is_late else 'onTime' - if not is_main: post_type = f"{post_type} bonus" + post_type = format_post_type(is_late, post.get('isMain')) # caption caption = post.get('caption') # latitude @@ -1247,192 +2280,819 @@ def bereal_posts(files_found, report_folder, seeker, wrap_text, timezone_offset) # longitude longitude = post.get('location', {}).get('longitude') # tags - tags = get_tags(post, html_format = False) - tags_html = get_tags(post, html_format = True) - # post visibilities - post_visibilities = get_post_visibilities(post) + tags = get_tags(post) + tags_html = get_tags(post, html_format=True) + # visibilities + visibilities = get_post_visibilities(post) + visibilities_html = get_post_visibilities(post, html_format=True) # taken at taken_at = convert_cocoa_core_data_ts_to_utc(post.get('takenAt')) # retake counter retake_counter = post.get('retakeCounter') # primary - p_mt, p_url, p_size = get_media(post.get('primaryMedia')) - p_url_html = generic_url(p_url, html_format = True) + primary = post.get('primaryMedia', {}) + p_mt, p_url, p_size = get_media(primary) + p_url_html = generic_url(p_url, html_format=True) + if p_mt == 'video': + p_media_ref_id = media_item_from_url(seeker, post.get('primaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) # secondary - s_mt, s_url, s_size = get_media(post.get('secondaryMedia')) - s_url_html = generic_url(s_url, html_format = True) - + secondary = post.get('secondaryMedia', {}) + s_mt, s_url, s_size = get_media(secondary) + s_url_html = generic_url(s_url, html_format=True) + if s_mt == 'video': + s_media_ref_id = media_item_from_url(seeker, post.get('secondaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # music + song_title = None + song_url = None + song_url_html = None + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}][{m}][posts][{p}]" # html row - data_list_html.append((taken_at, moment_at, post_type, author, p_mt, p_url_html, s_mt, s_url_html, - t_mt, t_url_html, caption, latitude, longitude, retake_counter, late_secs, tags_html, - moment_id, bereal_id, file_rel_path, location)) + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + None, None, None, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) # lava row - data_list.append((taken_at, moment_at, post_type, author, p_mt, p_url, s_mt, s_url, - t_mt, t_url, caption, latitude, longitude, retake_counter, late_secs, tags, - moment_id, bereal_id, file_rel_path, location)) + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + None, None, None, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # person posts + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/person\/profiles\/[\w-]+\?') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass + # user posts + user_posts = json_data.get('beRealOfTheDay', {}).get('userPosts') + if not bool(user_posts): + continue - # lava types - data_headers[0] = (data_headers[0], 'datetime') - data_headers[1] = (data_headers[1], 'datetime') + # author + author_id, author_user_name = get_user(user_posts) + author = format_userid(author_id, author_user_name) - # paths - source_path = ', '.join(source_paths) + # moment id + moment_id = user_posts.get('momentId') + if not bool(moment_id): moment_id = user_posts.get('moment', {}).get('id') - return data_headers, (data_list, data_list_html), source_path + # posts + posts = user_posts.get('posts') + if not bool(posts): + continue + + # array + for i in range(0, len(posts)): + device_file_paths = db_device_file_paths + + post = posts[i] + if not bool(post): + continue + + # bereal id + bereal_id = post.get('id') + # is late? + is_late = post.get('isLate') + # late in seconds + late_secs = get_late_secs(post) + # post type + post_type = format_post_type(is_late, post.get('isMain')) + # caption + caption = post.get('caption') + # latitude + latitude = post.get('location', {}).get('latitude') + # longitude + longitude = post.get('location', {}).get('longitude') + # visibilities + visibilities = get_post_visibilities(post) + visibilities_html = get_post_visibilities(post, html_format=True) + # taken at + taken_at = convert_iso8601_to_utc(post.get('takenAt')) + # retake counter + retake_counter = post.get('retakeCounter') + # primary + primary = post.get('primary', {}) + p_mt, p_url, p_size = get_media(primary) + p_url_html = generic_url(p_url, html_format=True) + if p_mt == 'video': + p_media_ref_id = media_item_from_url(seeker, post.get('primaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + secondary = post.get('secondary', {}) + s_mt, s_url, s_size = get_media(secondary) + s_url_html = generic_url(s_url, html_format=True) + if s_mt == 'video': + s_media_ref_id = media_item_from_url(seeker, post.get('secondaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # music + song_title = None + song_url = None + song_url_html = None + # tags + tags = get_tags(post) + tags_html = get_tags(post, html_format=True) + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[beRealOfTheDay][userPosts][posts][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + None, None, None, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + None, None, None, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Person Posts: " + str(ex)) + pass + + try: + # memories + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/memories-v(\d)+\/[\w-]+$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): + continue + + # posts + posts = json_data.get('posts') + if not bool(posts): + continue + + # array + for i in range(0, len(posts)): + device_file_paths = db_device_file_paths + + post = posts[i] + if not bool(post): + continue + + # moment id + moment_id = str(record[1]).split('/')[-1] + # bereal id + bereal_id = post.get('id') + # late in seconds + late_secs = get_late_secs(post) + # post type + post_type = format_post_type(post.get('isLate'), post.get('isMain')) + # caption + caption = post.get('caption') + # latitude + latitude = post.get('location', {}).get('latitude') + # longitude + longitude = post.get('location', {}).get('longitude') + # visibilities + visibilities = get_post_visibilities(post) + visibilities_html = get_post_visibilities(post, html_format=True) + # taken at + taken_at = convert_iso8601_to_utc(post.get('takenAt')) + # retake counter + retake_counter = post.get('retakeCounter') + # primary + primary = post.get('primary', {}) + p_mt, p_url, p_size = get_media(primary) + p_url_html = generic_url(p_url, html_format=True) + if p_mt == 'video': + p_media_ref_id = media_item_from_url(seeker, post.get('primaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + secondary = post.get('secondary', {}) + s_mt, s_url, s_size = get_media(secondary) + s_url_html = generic_url(s_url, html_format=True) + if s_mt == 'video': + s_media_ref_id = media_item_from_url(seeker, post.get('secondaryPlaceholder', {}).get('url'), artifact_info, from_photos=True) + else: + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # thumbnail + thumbnail = post.get('thumbnail', {}) + t_mt, t_url, t_size = get_media(thumbnail) + t_url_html = generic_url(t_url, html_format=True) + t_media_ref_id = media_item_from_url(seeker, t_url, artifact_info, from_photos = p_mt == 'video') + t_media_item = lava_get_full_media_info(t_media_ref_id) + if t_media_item: device_file_paths.append(get_device_file_path(t_media_item[5], seeker)) + # music + song_title = get_music_artist_and_track(post) + song_url = get_music_url(post) + song_url_html = get_music_url(post, html_format=True) + # tags + tags = get_tags(post) + tags_html = get_tags(post, html_format=True) + # author + author = format_userid(bereal_user_id) + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[posts][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + t_mt, t_url_html, t_media_ref_id, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + t_mt, t_url, t_media_ref_id, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Memories Posts: " + str(ex)) + pass + + # feeds discovery + try: + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/discovery$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # json + if not bool(json_data): + continue + + # posts + posts = json_data.get('posts') + if not bool(posts): + continue + + # array + for i in range(0, len(posts)): + device_file_paths = db_device_file_paths + + post = posts[i] + if not bool(post): + continue + + # moment id + moment_id = None + # bereal id + bereal_id = post.get('id') + # late in seconds + late_secs = get_late_secs(post) + # post type + post_type = format_post_type(post.get('mediaType') == 'late', True) + # caption + caption = post.get('caption') + # latitude + latitude = post.get('location', {}).get('_latitude') + # longitude + longitude = post.get('location', {}).get('_longitude') + # visibilities + visibilities = get_post_visibilities(post) + visibilities_html = get_post_visibilities(post, html_format=True) + # taken at + taken_at = convert_ts_int_to_utc(post.get('takenAt', {}).get('_seconds')) + # retake counter + retake_counter = post.get('retakeCounter') + # primary + p_mt = 'photo' + p_url = post.get('photoURL') + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + s_mt = 'photo' + s_url = post.get('secondaryPhotoURL') + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # music + song_title = get_music_artist_and_track(post) + song_url = get_music_url(post) + song_url_html = get_music_url(post, html_format=True) + # tags + tags = get_tags(post) + tags_html = get_tags(post, html_format=True) + # author + author_id, author_user_name = get_user(post) + author = format_userid(author_id, author_user_name) + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[posts][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + None, None, None, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + None, None, None, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Feeds Discovery: " + str(ex)) + pass + + # EntitiesStore.sqlite + elif file_rel_path.endswith('EntitiesStore.sqlite'): + try: + query = ''' + SELECT + P.Z_PK, + U.Z_PK, + PM.Z_PK, + SM.Z_PK, + PTM.Z_PK, + STM.Z_PK, + M.Z_PK, + L.Z_PK, + (P.ZTAKENAT + 978307200) AS "taken_at", + P.ZISLATE, + P.ZISMAIN, + U.ZID AS "author_id", + coalesce(U.ZFULLNAME, U.ZUSERNAME) AS "author_name", + PM.ZMEDIATYPE, + PM.ZURL, + SM.ZMEDIATYPE, + SM.ZURL, + PTM.ZMEDIATYPE, + PTM.ZURL, + STM.ZMEDIATYPE, + STM.ZURL, + (M.ZARTIST || " - " || M.ZTRACK) AS "song_title", + M.ZOPENURL, + P.ZCAPTION, + L.ZLATITUDE, + L.ZLONGITUDE, + P.ZRETAKECOUNTER, + P.ZLATEINSECONDS, + P.ZVISIBILITIES, + (P.ZRESHAREDFROM > 0) AS "reshared", + NULL AS "tags", + P.ZMOMENTID, + P.ZID AS "bereal_id" + FROM ZPOSTMO AS "P" + LEFT JOIN ZUSERMO AS "U" ON (P.ZUSER = U.Z_PK) + LEFT JOIN ZPOSTMO_MEDIAMO AS "PM" ON (P.ZPRIMARYMEDIA = PM.Z_PK) + LEFT JOIN ZPOSTMO_MEDIAMO AS "SM" ON (P.ZSECONDARYMEDIA = SM.Z_PK) + LEFT JOIN ZPOSTMO_MEDIAMO AS "PTM" ON (P.ZPRIMARYTHUMBNAIL = PTM.Z_PK) + LEFT JOIN ZPOSTMO_MEDIAMO AS "STM" ON (P.ZSECONDARYTHUMBNAIL = STM.Z_PK) + LEFT JOIN ZPOSTMO_MUSICMO AS "M" ON (P.ZMUSIC = M.Z_PK) + LEFT JOIN ZPOSTMO_LOCATIONMO AS "L" ON (P.ZLOCATION = L.Z_PK) + ''' + db_records = get_sqlite_db_records(file_found, query) + if len(db_records) == 0: + continue + + for record in db_records: + device_file_paths = [ device_file_path ] + + # taken at + taken_at = convert_ts_int_to_utc(record[8]) + # post type + post_type = format_post_type(record[9], record[10]) + # author + author = format_userid(record[11], record[12]) + # primary + p_mt = record[13] + p_url = generic_url(record[14]) + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + s_mt = record[15] + s_url = generic_url(record[16]) + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # primary thumbnail + t_mt = record[17] + t_url = generic_url(record[18]) + t_url_html = generic_url(t_url, html_format=True) + t_media_ref_id = media_item_from_url(seeker, t_url, artifact_info, from_photos = p_mt == 'video') + t_media_item = lava_get_full_media_info(t_media_ref_id) + if t_media_item: device_file_paths.append(get_device_file_path(t_media_item[5], seeker)) + # secondary thumbnail +# todo + # music + song_title = record[21] + song_url = generic_url(record[22]) + song_url_html = get_music_url(song_url, html_format=True) + # caption + caption = record[23] + # latitude + latitude = record[24] + # longitude + longitude = record[25] + # retake counter + retake_counter = record[26] + # late in seconds + late_secs = get_late_secs(record[27]) + # visibility + visibilities_plist = get_plist_content(record[28]) + visibilities = get_post_visibilities(visibilities_plist) + visibilities_html = get_post_visibilities(visibilities_plist, html_format=True) + # reshared + reshared = record[29] + # tags +# todo + tags = record[30] + # moment id + moment_id = record[31] + # bereal id + bereal_id = record[32] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"ZUSERMO (Z_PK: {record[0]})" ] + if record[1] is not None: location.append(f"ZPOSTMO_MEDIAMO (Z_PK: {record[1]})") + if record[2] is not None: location.append(f"ZPOSTMO_MEDIAMO (Z_PK: {record[2]})") + if record[3] is not None: location.append(f"ZPOSTMO_MEDIAMO (Z_PK: {record[3]})") + if record[4] is not None: location.append(f"ZPOSTMO_MEDIAMO (Z_PK: {record[4]})") + if record[5] is not None: location.append(f"ZPOSTMO_MUSICMO (Z_PK: {record[5]})") + if record[6] is not None: location.append(f"ZPOSTMO_LOCATIONMO (Z_PK: {record[6]})") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + t_mt, t_url_html, t_media_ref_id, song_title, song_url_html, caption, latitude, longitude, retake_counter, late_secs, + visibilities_html, tags_html, moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + t_mt, t_url, t_media_ref_id, song_title, song_url, caption, latitude, longitude, retake_counter, late_secs, + visibilities, tags, moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + return data_headers, (data_list, data_list_html), ' ' # pinned memories @artifact_processor def bereal_pinned_memories(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Pinned at', 'Moment day', 'Post type', 'Author ID', 'Primary media type', 'Primary URL', 'Secondary media type', 'Secondary URL', - 'Moment ID', 'Pin ID', 'Source file name', 'Location' ] + data_headers = ( + ('Pinned at', 'datetime'), + 'Post type', + 'Author', + 'Primary media type', + 'Primary URL', + ('Primary image', 'media', 'height: 96px; border-radius: 5%;'), + 'Secondary media type', + 'Secondary URL', + ('Secondary image', 'media', 'height: 96px; border-radius: 5%;'), + 'Moment ID', + 'BeReal ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() - - def append_data(obj, index, from_person): - if not bool(obj): - return - - # taken at - taken_at = convert_cocoa_core_data_ts_to_utc(obj.get('takenAt')) - # moment day - moment_day = convert_cocoa_core_data_ts_to_utc(obj.get('momentDay')) - # post type - post_type = obj.get('analyticsPostType') - # author id - author_id = obj.get('analyticsAuthorID') - # primary - p_mt, p_url, p_size = get_media(obj.get('primary')) - p_url_html = generic_url(p_url, html_format = True) - # secondary - s_mt, s_url, s_size = get_media(obj.get('secondary')) - s_url_html = generic_url(s_url, html_format = True) - # moment id - moment_id = obj.get('analyticsMomentID') - # unique id - pin_id = obj.get('id') - - - # location - location = f"[object][{author_id}][{index}]" if from_person else f"[object][{index}]" - - # html row - data_list_html.append((taken_at, moment_day, post_type, author_id, p_mt, p_url_html, s_mt, s_url_html, - moment_id, pin_id, file_rel_path, location)) - # lava row - data_list.append((taken_at, moment_day, post_type, author_id, p_mt, p_url, s_mt, s_url, - moment_id, pin_id, file_rel_path, location)) - - return - + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_pinned_memories']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) + + # disk-bereal-MemoriesRepository-pinnedMemories-key + # disk-bereal-PersonRepository-pinnedMemories-key + if file_rel_path.startswith('disk-bereal-MemoriesRepository-pinnedMemories-key') or file_rel_path.startswith('disk-bereal-PersonRepository-pinnedMemories-key'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue - # object? - obj_ref = json_data.get('object') - if not bool(obj_ref): - continue + # disk-bereal-MemoriesRepository-pinnedMemories-key + if file_rel_path.startswith('disk-bereal-MemoriesRepository-pinnedMemories-key'): + # pinned + for i in range(0, len(obj_ref)): + device_file_paths = [ device_file_path ] - # memories repository - # disk-bereal-MemoriesRepository-pinnedMemories-key - if file_rel_path.startswith('disk-bereal-MemoriesRepository-pinnedMemories-key'): - # pinned - for i in range(0, len(obj_ref)): - append_data(obj_ref[i], i, from_person = False) + pin = obj_ref[i] + if not bool(pin): + continue - # person memories - # disk-bereal-PersonRepository-pinnedMemories-key - elif file_rel_path.startswith('disk-bereal-PersonRepository-pinnedMemories-key'): + # taken at + taken_at = convert_cocoa_core_data_ts_to_utc(pin.get('takenAt')) + # post type + post_type = pin.get('analyticsPostType') + # author + author = format_userid(pin.get('analyticsAuthorID')) + # primary + p_mt, p_url, p_size = get_media(pin.get('primary')) + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + s_mt, s_url, s_size = get_media(pin.get('secondary')) + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # moment id + moment_id = pin.get('analyticsMomentID') + # bereal id + bereal_id = pin.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][{i}]" + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-PersonRepository-pinnedMemories-key + elif file_rel_path.startswith('disk-bereal-PersonRepository-pinnedMemories-key'): + try: # users for key, val in obj_ref.items(): # uids if not bool(val) or not isinstance(val, list): continue - + # pinned for i in range(0, len(val)): - append_data(val[i], i, from_person = True) + device_file_paths = [ device_file_path ] - # none - else: - continue - - except Exception as e: - logfunc(f"Error: {str(e)}") - pass + pin = val[i] + if not bool(pin): + continue + + # taken at + taken_at = convert_cocoa_core_data_ts_to_utc(pin.get('takenAt')) + # post type + post_type = pin.get('analyticsPostType') + # author + author_id = pin.get('analyticsAuthorID') + author = format_userid(author_id) + # primary + p_mt, p_url, p_size = get_media(pin.get('primary')) + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + s_mt, s_url, s_size = get_media(pin.get('secondary')) + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # moment id + moment_id = pin.get('analyticsMomentID') + # bereal id + bereal_id = pin.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][{author_id}][{i}]" - # lava types - data_headers[0] = (data_headers[0], 'datetime') - data_headers[1] = (data_headers[1], 'datetime') + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # pinned memories + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/memories-v(\d)+\/pinned-memories\/for-user\/') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) == 0: + continue - # paths - source_path = ', '.join(source_paths) + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + obj_ref = json_data.get('pinnedMemories') + if not bool(obj_ref): + continue - return data_headers, (data_list, data_list_html), source_path + # user id + user_id = str(record[1]).split('/')[-1] + + # pinned + for i in range(0, len(obj_ref)): + device_file_paths = db_device_file_paths + + pin = obj_ref[i] + if not pin: + continue + + # taken at + taken_at = convert_iso8601_to_utc(pin.get('takenAt')) + # post type + post_type = format_post_type(pin.get('isLate'), pin.get('isMain')) + # author + author = format_userid(user_id) + # primary + p_mt, p_url, p_size = get_media(pin.get('primary')) + p_url_html = generic_url(p_url, html_format=True) + p_media_ref_id = media_item_from_url(seeker, p_url, artifact_info) + p_media_item = lava_get_full_media_info(p_media_ref_id) + if p_media_item: device_file_paths.append(get_device_file_path(p_media_item[5], seeker)) + # secondary + s_mt, s_url, s_size = get_media(pin.get('secondary')) + s_url_html = generic_url(s_url, html_format=True) + s_media_ref_id = media_item_from_url(seeker, s_url, artifact_info) + s_media_item = lava_get_full_media_info(s_media_ref_id) + if s_media_item: device_file_paths.append(get_device_file_path(s_media_item[5], seeker)) + # moment id + moment_id = pin.get('momentId') + # bereal id + bereal_id = pin.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[pinnedMemories][{i}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((taken_at, post_type, author, p_mt, p_url_html, p_media_ref_id, s_mt, s_url_html, s_media_ref_id, + moment_id, bereal_id, source_file_name_html, location)) + # lava row + data_list.append((taken_at, post_type, author, p_mt, p_url, p_media_ref_id, s_mt, s_url, s_media_ref_id, + moment_id, bereal_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Pinned Memories: " + str(ex)) + pass + + return data_headers, (data_list, data_list_html), ' ' # realmojis @artifact_processor def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Created', 'BeReal ID', 'Direction', 'Owner', 'Author', 'Emoji', 'RealMoji', 'Moment ID', 'RealMoji ID', 'Source file name', 'Location' ] + data_headers = ( + ('Created', 'datetime'), + 'BeReal ID', + 'Direction', + 'Owner', + 'Author', + 'Emoji', + 'RealMoji URL', + ('RealMoji picture', 'media', 'height: 96px; border-radius: 50%;'), + 'Moment ID', + 'RealMoji ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_paths = set() + device_file_paths = [] + artifact_info = inspect.stack()[0] + artifact_info_name = __artifacts_v2__['bereal_realmojis']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue - - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) - - # object? - obj_ref = json_data.get('object') - if not bool(obj_ref): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) + + # person (dictionary) + # PersonRepository + if file_rel_path.startswith('PersonRepository'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - # person (dictionary) - # PersonRepository - if file_rel_path.startswith('PersonRepository'): # series series = json_data.get('object', {}).get('beRealOfTheDay', {}).get('series') if not bool(series): continue - # owner: user id, user name - owner_user_id, owner_user_name = get_user(series) # owner + owner_user_id, owner_user_name = get_user(series) owner = format_userid(owner_user_id, owner_user_name) # posts @@ -1457,6 +3117,8 @@ def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_off continue for r in range(0, len(realmojis)): + device_file_paths = [ device_file_path ] + # realmoji realmoji = realmojis[r] if not bool(realmoji): @@ -1473,33 +3135,49 @@ def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_off direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' # emoji emoji = realmoji.get('emoji') - uri_moji = generic_url(realmoji.get('uri'), False) - uri_moji_html = generic_url(realmoji.get('uri'), True) + uri_moji = generic_url(realmoji.get('uri')) + uri_moji_html = generic_url(realmoji.get('uri'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) # realmoji id realmoji_id = realmoji.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][beRealOfTheDay][series][posts][{i}][realMojis][{r}]" # html row - data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, moment_id, realmoji_id, file_rel_path, location)) - + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) # lava row - data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, moment_id, realmoji_id, file_rel_path, location)) - - # memories (array) - # disk-bereal-MemoriesRepository-subject-key - elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # memories (array) + # disk-bereal-MemoriesRepository-subject-key + elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + # memories memories = json_data.get('object') if not bool(memories): continue - # owner user id - owner_user_id = bereal_user_id - # owner user name - owner_user_name = map_id_name.get(owner_user_id) # owner + owner_user_id = bereal_user_id + owner_user_name = bereal_user_map.get(owner_user_id) owner = format_userid(owner_user_id, owner_user_name) # array @@ -1508,89 +3186,404 @@ def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_off if not bool(memory): continue - # detailed posts - detailed_posts = memory.get('allDetailedPosts') - if not bool(detailed_posts): - continue - for j in range(0, len(detailed_posts)): - # detailed post - detailed_post = detailed_posts[j] - if not bool(detailed_post): - continue + # detailed posts + detailed_posts = memory.get('allDetailedPosts') + if not bool(detailed_posts): + continue + for j in range(0, len(detailed_posts)): + # detailed post + detailed_post = detailed_posts[j] + if not bool(detailed_post): + continue + + # bereal id + bereal_id = detailed_post.get('id') + # moment id + moment_id = detailed_post.get('momentID') + + # metadata + metadata = detailed_post.get('details', {}).get('full', {}).get('metadata') + if not bool(metadata): + continue + + # realmojis + realmojis = metadata['realMojis'] + if not bool(realmojis): + continue + + for r in range(0, len(realmojis)): + device_file_paths = [ device_file_path ] + + # realmoji + realmoji = realmojis[r] + if not bool(realmoji): + continue + + # reaction date + reaction_date = realmoji.get('date') + reaction_date = convert_cocoa_core_data_ts_to_utc(reaction_date) + # author: id, user name + author_id, author_user_name = get_user(realmoji) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # emoji + emoji = realmoji.get('emoji') + uri_moji = generic_url(realmoji.get('uri')) + uri_moji_html = generic_url(realmoji.get('uri'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) + # realmoji id + realmoji_id = realmoji.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][{i}][allDetailedPosts][{j}][details][full][metadata][realMojis][{r}]" + + # html row + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) + # lava row + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-Production_postFeedItems + elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue + + # array (string, array) + for i in range(0, len(obj_ref)): + # array + if not (bool(obj_ref[i]) and isinstance(obj_ref[i], list)): + continue + + # moment + for m in range(0, len(obj_ref[i])): + moment = obj_ref[i][m] + if not bool(moment): + continue + + posts = moment.get('posts') + if not bool(posts): + continue + + for p in range(0, len(posts)): + post = posts[p] + if not bool(post): + continue + + # bereal id + bereal_id = post.get('id') + # moment id + moment_id = post.get('momentID') + # owner + owner_user_id, owner_user_name = get_user(post) + owner = format_userid(owner_user_id, owner_user_name) + + # realmojis + realmojis = post.get('realMojis') + if not bool(realmojis): + continue + + for r in range(0, len(realmojis)): + device_file_paths = [ device_file_path ] + + # realmoji + realmoji = realmojis[r] + if not bool(realmoji): + continue + + # reaction date + reaction_date = realmoji.get('date') + reaction_date = convert_cocoa_core_data_ts_to_utc(reaction_date) + # author: id, user name + author_id, author_user_name = get_user(realmoji) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # emoji + emoji = realmoji.get('emoji') + uri_moji = generic_url(realmoji.get('uri')) + uri_moji_html = generic_url(realmoji.get('uri'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) + # realmoji id + realmoji_id = realmoji.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"[object][beRealOfTheDay][series][posts][{i}][realMojis][{r}]" + + # html row + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) + # lava row + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # person posts + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/person\/profiles\/[\w-]+\?') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): + continue + + # user posts + user_posts = json_data.get('beRealOfTheDay', {}).get('userPosts') + if not bool(user_posts): + continue + + # owner + owner_id, owner_user_name = get_user(user_posts) + owner = format_userid(owner_id, owner_user_name) + + # moment id + moment_id = user_posts.get('momentId') + if not bool(moment_id): moment_id = user_posts.get('moment', {}).get('id') + + # posts + posts = user_posts.get('posts') + if not bool(posts): + continue + + # array + for i in range(0, len(posts)): + post = posts[i] + if not bool(post): + continue + + # bereal id (thread) + bereal_id = post.get('id') + + # realmojis + realmojis = post.get('realMojis') + if not bool(realmojis): + continue + + for r in range(0, len(realmojis)): + device_file_paths = db_device_file_paths + + # realmoji + realmoji = realmojis[r] + if not bool(realmoji): + continue + + # reaction date + reaction_date = convert_iso8601_to_utc(realmoji.get('postedAt')) + # author: id, user name + author_id, author_user_name = get_user(realmoji) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # emoji + emoji = realmoji.get('emoji') + uri_moji = generic_url(realmoji.get('media', {}).get('url')) + uri_moji_html = generic_url(realmoji.get('media', {}).get('url'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) + # realmoji id + realmoji_id = realmoji.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[beRealOfTheDay][userPosts][posts][{i}][realMojis][{r}]") + location = COMMA_SEP.join(location) - # bereal id - bereal_id = detailed_post.get('id') - # moment id - moment_id = detailed_post.get('momentID') + # html row + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) + # lava row + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Persons: " + str(ex)) + pass - # metadata - metadata = detailed_post.get('details', {}).get('full', {}).get('metadata') - if not bool(metadata): + try: + # memories + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/memories-v(\d)+\/[\w-]+$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): continue - # realmojis - realmojis = metadata['realMojis'] - if not bool(realmojis): + # posts + posts = json_data.get('posts') + if not bool(posts): continue - for r in range(0, len(realmojis)): - # realmoji - realmoji = realmojis[r] - if not bool(realmoji): + # array + for i in range(0, len(posts)): + post = posts[i] + if not bool(post): continue - # reaction date - reaction_date = realmoji.get('date') - reaction_date = convert_cocoa_core_data_ts_to_utc(reaction_date) - # author: id, user name - author_id, author_user_name = get_user(realmoji) - # author - author = format_userid(author_id, author_user_name) - # direction - direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' - # emoji - emoji = realmoji.get('emoji') - uri_moji = generic_url(realmoji.get('uri'), False) - uri_moji_html = generic_url(realmoji.get('uri'), True) - # realmoji id - realmoji_id = realmoji.get('id') + # owner + owner = format_userid(bereal_user_id) + # moment id + moment_id = str(record[1]).split('/')[-1] + # bereal id + bereal_id = post.get('id') - # location - location = f"[object][{i}][allDetailedPosts][{j}][details][full][metadata][realMojis][{r}]" + # realmojis + realmojis = post.get('realmojis') + if not bool(realmojis): + continue - # html row - data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, moment_id, realmoji_id, file_rel_path, location)) - - # lava row - data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, moment_id, realmoji_id, file_rel_path, location)) + for r in range(0, len(realmojis)): + device_file_paths = db_device_file_paths - # disk-bereal-Production_postFeedItems - elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): - # array (string, array) - for i in range(0, len(obj_ref)): - # array - if not (bool(obj_ref[i]) and isinstance(obj_ref[i], list)): - continue + # realmoji + realmoji = realmojis[r] + if not bool(realmoji): + continue - # moment - for m in range(0, len(obj_ref[i])): - moment = obj_ref[i][m] - if not bool(moment): + # reaction date + reaction_date = convert_iso8601_to_utc(realmoji.get('postedAt')) + # author: id, user name + author_id, author_user_name = get_user(realmoji) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # emoji + emoji = realmoji.get('emoji') + uri_moji = generic_url(realmoji.get('media', {}).get('url')) + uri_moji_html = generic_url(realmoji.get('media', {}).get('url'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) + # realmoji id + realmoji_id = realmoji.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[posts][{i}][realmojis][{r}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) + # lava row + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Memories: " + str(ex)) + pass + + try: + # feeds discovery + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/discovery$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # json + if not bool(json_data): continue - posts = moment.get('posts') + # posts + posts = json_data.get('posts') if not bool(posts): continue - for p in range(0, len(posts)): - post = posts[p] + # array + for i in range(0, len(posts)): + post = posts[i] if not bool(post): continue - # owner: user id, user name - owner_user_id, owner_user_name = get_user(post) # owner - owner = format_userid(owner_user_id, owner_user_name) + owner_id, owner_user_name = get_user(post) + owner = format_userid(owner_id, owner_user_name) + # moment id + moment_id = None + # bereal id + bereal_id = post.get('id') # realmojis realmojis = post.get('realMojis') @@ -1598,14 +3591,15 @@ def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_off continue for r in range(0, len(realmojis)): + device_file_paths = db_device_file_paths + # realmoji realmoji = realmojis[r] if not bool(realmoji): continue # reaction date - reaction_date = realmoji.get('date') - reaction_date = convert_cocoa_core_data_ts_to_utc(reaction_date) + reaction_date = convert_ts_int_to_utc(realmoji.get('date', {}).get('_seconds')) # author: id, user name author_id, author_user_name = get_user(realmoji) # author @@ -1614,64 +3608,145 @@ def bereal_realmojis(files_found, report_folder, seeker, wrap_text, timezone_off direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' # emoji emoji = realmoji.get('emoji') - uri_moji = generic_url(realmoji.get('uri'), False) - uri_moji_html = generic_url(realmoji.get('uri'), True) + uri_moji = generic_url(realmoji.get('uri')) + uri_moji_html = generic_url(realmoji.get('uri'), html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) # realmoji id realmoji_id = realmoji.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location - location = f"[object][beRealOfTheDay][series][posts][{i}][realMojis][{r}]" + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[posts][{i}][realMojis][{r}]") + location = COMMA_SEP.join(location) # html row - data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, moment_id, realmoji_id, file_rel_path, location)) - + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) # lava row - data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, moment_id, realmoji_id, file_rel_path, location)) + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Feeds Discovery: " + str(ex)) + pass + + # EntitiesStore.sqlite + elif file_rel_path.endswith('EntitiesStore.sqlite'): + try: + query = ''' + SELECT + R.Z_PK, + P.Z_PK, + U.Z_PK, + (R.ZDATE + 978307200), + P.ZID AS "bereal_id", + IIF(U.ZID = R.ZUSERID, "Outgoing", "Incoming"), + U.ZID AS "owner_id", + U.ZUSERNAME AS "owner_name", + R.ZUSERID AS "author_id", + R.ZUSERNAME AS "author_name", + R.ZEMOJI, + R.ZURL, + P.ZMOMENTID AS "moment_id", + R.ZID AS "realmoji_id" + FROM ZPOSTMO_REALMOJIMO AS "R" + LEFT JOIN ZPOSTMO AS "P" ON (R.ZPOST = P.Z_PK) + LEFT JOIN ZUSERMO AS "U" ON (P.ZUSER = U.Z_PK) + ''' + db_records = get_sqlite_db_records(file_found, query) + if len(db_records) == 0: + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass - - # lava types - data_headers[0] = (data_headers[0], 'datetime') + for record in db_records: + device_file_paths = [ device_file_path ] + + # reaction date + reaction_date = convert_ts_int_to_utc(record[3]) + # bereal id + bereal_id = record[4] + # direction + direction = record[5] + # owner + owner = format_userid(record[6], record[7]) + # author + author = format_userid(record[8], record[9]) + # emoji + emoji = record[10] + uri_moji = generic_url(record[11]) + uri_moji_html = generic_url(record[11], html_format=True) + # realmoji + r_media_ref_id = media_item_from_url(seeker, uri_moji, artifact_info) + r_media_item = lava_get_full_media_info(r_media_ref_id) + if r_media_item: device_file_paths.append(get_device_file_path(r_media_item[5], seeker)) + # moment id + moment_id = record[12] + # realmoji id + realmoji_id = record[13] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"ZPOSTMO_REALMOJIMO (Z_PK: {record[0]})" ] + if record[1] is not None: location.append(f"ZPOSTMO (Z_PK: {record[1]})") + if record[2] is not None: location.append(f"ZUSERMO (Z_PK: {record[2]})") + location = COMMA_SEP.join(location) - # paths - source_path = ', '.join(source_paths) + # html row + data_list_html.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji_html, r_media_ref_id, + moment_id, realmoji_id, source_file_name_html, location)) + # lava row + data_list.append((reaction_date, bereal_id, direction, owner, author, emoji, uri_moji, r_media_ref_id, + moment_id, realmoji_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' # comments @artifact_processor def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Created', 'BeReal ID', 'Direction', 'Owner', 'Author', 'Text', 'Moment ID', 'Comment ID', 'Source file name', 'Location' ] + data_headers = ( + ('Created', 'datetime'), + 'BeReal ID', + 'Direction', + 'Owner', + 'Author', + 'Text', + 'Moment ID', + 'Comment ID', + 'Source file name', + 'Location' + ) data_list = [] - source_paths = set() + data_list_html = [] + device_file_paths = [] + artifact_info_name = __artifacts_v2__['bereal_comments']['name'] # all files for file_found in files_found: - json_data = get_json_data(file_found) - if not bool(json_data): - continue - - try: - if is_platform_windows() and file_found.startswith('\\\\?\\'): - file_location = str(Path(file_found[4:]).parents[1]) - file_rel_path = str(Path(file_found[4:]).relative_to(file_location)) - else: - file_location = str(Path(file_found).parents[1]) - file_rel_path = str(Path(file_found).relative_to(file_location)) - source_paths.add(file_location) - - # object? - obj_ref = json_data.get('object') - if not bool(obj_ref): - continue + file_rel_path = Path(Path(file_found).parent.name, Path(file_found).name).as_posix() + device_file_path = get_device_file_path(file_found, seeker) + + # person (dictionary) + # PersonRepository + if file_rel_path.startswith('PersonRepository'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - # person (dictionary) - # PersonRepository - if file_rel_path.startswith('PersonRepository'): # series series = json_data.get('object', {}).get('beRealOfTheDay', {}).get('series') if not bool(series): @@ -1703,6 +3778,8 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs continue for c in range(0, len(comments)): + device_file_paths = [ device_file_path ] + # comment comment = comments[c] if not bool(comment): @@ -1722,15 +3799,31 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs # comment id comment_id = comment.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][beRealOfTheDay][series][posts][{i}][comment][{c}]" + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) # lava row - data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, file_rel_path, location)) + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # memories (array) + # disk-bereal-MemoriesRepository-subject-key + elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue - # memories (array) - # disk-bereal-MemoriesRepository-subject-key - elif file_rel_path.startswith('disk-bereal-MemoriesRepository-subject-key'): # memories memories = json_data.get('object') if not bool(memories): @@ -1739,7 +3832,7 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs # owner user id owner_user_id = bereal_user_id # owner user name - owner_user_name = map_id_name.get(owner_user_id) + owner_user_name = bereal_user_map.get(owner_user_id) # owner owner = format_userid(owner_user_id, owner_user_name) @@ -1753,6 +3846,7 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs detailed_posts = memory.get('allDetailedPosts') if not bool(detailed_posts): continue + for j in range(0, len(detailed_posts)): # detailed post detailed_post = detailed_posts[j] @@ -1772,7 +3866,10 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs comments = metadata['comments'] if not bool(comments): continue + for c in range(0, len(comments)): + device_file_paths = [ device_file_path ] + # comment comment = comments[c] if not bool(comment): @@ -1792,14 +3889,35 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs # uid comment_id = comment.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}][allDetailedPosts][{j}][details][full][metadata][comments][{c}]" + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) # lava row - data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, file_rel_path, location)) + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # disk-bereal-Production_postFeedItems + elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): + try: + # json + json_data = get_json_file_content(file_found) + if not bool(json_data): + continue + + # object? + obj_ref = json_data.get('object') + if not bool(obj_ref): + continue - # disk-bereal-Production_postFeedItems - elif file_rel_path.startswith('disk-bereal-Production_postFeedItems'): # array (string, array) for i in range(0, len(obj_ref)): # array @@ -1825,7 +3943,6 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs owner_user_id, owner_user_name = get_user(post) # owner owner = format_userid(owner_user_id, owner_user_name) - # bereal id (thread) bereal_id = post.get('id') # moment id @@ -1835,6 +3952,8 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs if not bool(comments): continue for c in range(0, len(comments)): + device_file_paths = [ device_file_path ] + # comment comment = comments[c] if not bool(comment): @@ -1853,40 +3972,302 @@ def bereal_comments(files_found, report_folder, seeker, wrap_text, timezone_offs # uid comment_id = comment.get('id') + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location location = f"[object][{i}][{m}][posts][{p}][comment][{c}]" + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) # lava row - data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, file_rel_path, location)) + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + # Cache.db + elif file_rel_path.endswith('Cache.db'): + try: + # person posts + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/person\/profiles\/[\w-]+\?') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): + continue - except Exception as e: - logfunc(f"Error: {str(e)}") - pass - - # lava types - data_headers[0] = (data_headers[0], 'datetime') + # user posts + user_posts = json_data.get('beRealOfTheDay', {}).get('userPosts') + if not bool(user_posts): + continue + + # owner + owner_user_id, owner_user_name = get_user(user_posts) + owner = format_userid(owner_user_id, owner_user_name) + + # moment id + moment_id = user_posts.get('momentId') + if not bool(moment_id): moment_id = user_posts.get('moment', {}).get('id') + + # posts + posts = user_posts.get('posts') + if not bool(posts): + continue + + # array + for i in range(0, len(posts)): + post = posts[i] + if not bool(post): + continue + + # bereal id (thread) + bereal_id = post.get('id') + + # comments + comments = post.get('comments') + if not bool(comments): + continue + + for c in range(0, len(comments)): + device_file_paths = db_device_file_paths + + # comment + comment = comments[c] + if not bool(comment): + continue + + # creation date + creation_date = convert_iso8601_to_utc(comment.get('postedAt')) + # author: id, user name + author_id, author_user_name = get_user(comment) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # text + text = comment.get('content') + # uid + comment_id = comment.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[beRealOfTheDay][userPosts][posts][{i}][comments][{c}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) + # lava row + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Persons: " + str(ex)) + pass + + try: + # memories + query = BEREAL_CACHE_DB_QUERY.format(r'https:\/\/mobile[-\w]*\.bereal\.com\/api\/feeds\/memories-v(\d)+\/[\w-]+$') + db_records = get_sqlite_db_records_regexpr(file_found, query, regexpr=True) + if len(db_records) > 0: + for record in db_records: + db_device_file_paths = [ device_file_path ] + + # from file? + isDataOnFS = bool(record[2]) + # from FS + if isDataOnFS: + fs_cached_data_path = get_cache_db_fs_path(record[3], file_found, seeker) + json_data = get_json_file_content(fs_cached_data_path) + if bool(fs_cached_data_path): db_device_file_paths.append(get_device_file_path(fs_cached_data_path, seeker)) + # from BLOB + else: + json_data = get_json_content(record[3]) + + # object? + if not bool(json_data): + continue + + # posts + posts = json_data.get('posts') + if not bool(posts): + continue - # paths - source_path = ', '.join(source_paths) + # array + for i in range(0, len(posts)): + post = posts[i] + if not bool(post): + continue + + # moment id + moment_id = str(record[1]).split('/')[-1] + # bereal id + bereal_id = post.get('id') + # owner + owner_user_id = bereal_user_id + owner = format_userid(owner_user_id) + + # comments + comments = post.get('comments') + if not bool(comments): + continue + + for c in range(0, len(comments)): + device_file_paths = db_device_file_paths + + # comment + comment = comments[c] + if not bool(comment): + continue + + # creation date + creation_date = convert_iso8601_to_utc(comment.get('postedAt')) + # author: id, user name + author_id, author_user_name = get_user(comment) + # author + author = format_userid(author_id, author_user_name) + # direction + direction = 'Outgoing' if author_id == owner_user_id else 'Incoming' + # text + text = comment.get('content') + # uid + comment_id = comment.get('id') + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"cfurl_cache_receiver_data (entry_ID: {record[0]})" ] + location.append(f"[posts][{i}][comments][{c}]") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) + # lava row + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - Cached.db Memories: " + str(ex)) + pass + + # EntitiesStore.sqlite + elif file_rel_path.endswith('EntitiesStore.sqlite'): + try: + query = ''' + SELECT + C.Z_PK, + P.Z_PK, + U.Z_PK, + (C.ZCREATIONDATE + 978307200), + P.ZID AS "bereal_id", + IIF(U.ZID = C.ZUSERID, "Outgoing", "Incoming"), + U.ZID AS "owner_id", + U.ZUSERNAME AS "owner_name", + C.ZUSERID AS "author_id", + C.ZUSERNAME AS "author_name", + C.ZTEXT, + P.ZMOMENTID AS "moment_id", + C.ZID AS "comment_id" + FROM ZPOSTMO_COMMENTMO AS "C" + LEFT JOIN ZPOSTMO AS "P" ON (C.ZPOST = P.Z_PK) + LEFT JOIN ZUSERMO AS "U" ON (P.ZUSER = U.Z_PK) + ''' + db_records = get_sqlite_db_records(file_found, query) + if len(db_records) == 0: + continue + + for record in db_records: + device_file_paths = [ device_file_path ] + + # creation date + creation_date = convert_ts_int_to_utc(record[3]) + # bereal id + bereal_id = record[4] + # direction + direction = record[5] + # owner + owner = format_userid(record[6], record[7]) + # author + author = format_userid(record[8], record[9]) + # text + text = record[10] + # moment id + moment_id = record[11] + # comment id + comment_id = record[12] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = [ f"ZPOSTMO_COMMENTMO (Z_PK: {record[0]})" ] + if record[1] is not None: location.append(f"ZPOSTMO (Z_PK: {record[1]})") + if record[2] is not None: location.append(f"ZUSERMO (Z_PK: {record[2]})") + location = COMMA_SEP.join(location) + + # html row + data_list_html.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name_html, location)) + # lava row + data_list.append((creation_date, bereal_id, direction, owner, author, text, moment_id, comment_id, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass - return data_headers, data_list, source_path + return data_headers, (data_list, data_list_html), ' ' # messages @artifact_processor def bereal_messages(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Sent', 'Owner', 'Direction', 'Sender', 'Recipient', 'Message', 'Message type', - 'Source', 'Media URL', 'Thread ID', 'Location' ] + data_headers = ( + ('Sent', 'datetime'), + 'Owner', + 'Direction', + 'Sender', + 'Recipient', + 'Message', + 'Message type', + 'Source', + 'Media URL', + ('Media image', 'media', 'height: 96px; border-radius: 5%;'), + 'Thread ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_path = get_file_path(files_found, "bereal-chat.sqlite") + artifact_info = inspect.stack()[0] + device_file_paths = [] + file_found = get_file_path(files_found, "bereal-chat.sqlite") + device_file_path = get_device_file_path(file_found, seeker) query = ''' SELECT M.Z_PK, C.Z_PK, - ME.Z_PK, + ME.Z_PK, (M.ZCREATEDAT + 978307200) AS "sent", C.ZOWNER, IIF(M.ZSENDER = C.ZOWNER, "Outgoing", "Incoming") AS "direction", @@ -1907,64 +4288,94 @@ def bereal_messages(files_found, report_folder, seeker, wrap_text, timezone_offs LEFT JOIN ZMEDIAMO AS "ME" ON (M.Z_PK = ME.ZMESSAGE) ''' - db_records = get_sqlite_db_records(source_path, query) + db_records = get_sqlite_db_records(file_found, query) for record in db_records: + device_file_paths = [ device_file_path ] + # created created = convert_unix_ts_to_utc(record[3]) - # owner owner = record[4] - # sender sender = record[6] - # recipient if record[5] == 'Outgoing': recipient = bereal_user_id else: - recipient = owner - + recipient = owner # "id (fullame|userid)" owner = format_userid(owner) sender = format_userid(sender) recipient = format_userid(recipient) - # body body = record[7].decode('utf-8') if bool(record[7]) else record[7] - - # media url - media_url_html = generic_url(record[10], html_format = True) - media_url = generic_url(record[10], html_format = False) - + # media source + if record[9] == 'remote': + # media url + media_url = generic_url(record[10]) + media_url_html = generic_url(record[10], html_format=True) + # media + media_ref_id = media_item_from_url(seeker, media_url, artifact_info) + media_item = lava_get_full_media_info(media_ref_id) + if media_item: device_file_paths.append(get_device_file_path(media_item[5], seeker)) + elif record[9] == 'local': + # local path /Documents/ + media_url = record[10] + media_url_html = record[10] + # media + file_image = f"*/{bereal_app_identifier}/Documents/{record[10]}" + media_ref_id = check_in_media(seeker, file_image, artifact_info) + media_item = lava_get_full_media_info(media_ref_id) + if media_item: device_file_paths.append(get_device_file_path(media_item[5], seeker)) + else: + media_url = record[10] + media_url_html = record[10] + media_ref_id = None + media_item = None + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location - location = [ f'ZMESSAGEMO (Z_PK: {record[0]})' ] - if record[1] is not None: location.append(f'ZCONVERSATIONMO (Z_PK: {record[1]})') - if record[2] is not None: location.append(f'ZMEDIAMO (Z_PK: {record[2]})') - location = ', '.join(location) + location = [ f"ZMESSAGEMO (Z_PK: {record[0]})" ] + if record[1] is not None: location.append(f"ZCONVERSATIONMO (Z_PK: {record[1]})") + if record[2] is not None: location.append(f"ZMEDIAMO (Z_PK: {record[2]})") + location = COMMA_SEP.join(location) # html row - data_list_html.append((created, owner, record[5], sender, recipient, body, record[8], - record[9], media_url_html, record[11], location)) - + data_list_html.append((created, owner, record[5], sender, recipient, body, record[8], + record[9], media_url_html, media_ref_id, record[11], source_file_name_html, location)) # lava row - data_list.append((created, owner, record[5], sender, recipient, body, record[8], - record[9], media_url, record[11], location)) - - # lava types - data_headers[0] = (data_headers[0], 'datetime') + data_list.append((created, owner, record[5], sender, recipient, body, record[8], + record[9], media_url, media_ref_id, record[11], source_file_name, location)) - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' # chat list @artifact_processor def bereal_chat_list(files_found, report_folder, seeker, wrap_text, timezone_offset): - data_headers = [ 'Created', 'Owner', 'Type', 'Administrators', 'Participants', 'Last message', 'Total messages', 'Last updated', - 'Unread messages count', 'ID', 'Location' ] + data_headers = ( + ('Created', 'datetime'), + 'Owner', + 'Type', + 'Administrators', + 'Participants', + 'Last message', + 'Total messages', + ('Last updated', 'datetime'), + 'Unread messages count', + 'ID', + 'Source file name', + 'Location' + ) data_list = [] data_list_html = [] - source_path = get_file_path(files_found, "bereal-chat.sqlite") + device_file_paths = [] + file_found = get_file_path(files_found, "bereal-chat.sqlite") + device_file_path = get_device_file_path(file_found, seeker) query = ''' SELECT @@ -1986,54 +4397,50 @@ def bereal_chat_list(files_found, report_folder, seeker, wrap_text, timezone_off LEFT JOIN ZMESSAGEMO AS "M" ON (C.ZLASTMESSAGE = M.Z_PK) ''' - db_records = get_sqlite_db_records(source_path, query) + db_records = get_sqlite_db_records(file_found, query) for record in db_records: + device_file_paths = [ device_file_path ] + # created created = convert_unix_ts_to_utc(record[3]) - # owner owner = format_userid(record[4]) - # last updated last_updated = convert_unix_ts_to_utc(record[10]) - # admins (plist) admins = get_plist_content(record[6]) admins_list = [] for a in admins: admins_list.append(format_userid(a)) if bool(admins_list): - admins = '\n'.join(admins_list) - admins_html = admins.replace('\n', '
') - + admins = LINE_BREAK.join(admins_list) + admins_html = admins.replace(LINE_BREAK, HTML_LINE_BREAK) # participants (plist) participants = get_plist_content(record[7]) participants_list = [] for p in participants: participants_list.append(format_userid(p)) if bool(participants_list): - participants = '\n'.join(participants_list) - participants_html = participants.replace('\n', '
') - + participants = LINE_BREAK.join(participants_list) + participants_html = participants.replace(LINE_BREAK, HTML_LINE_BREAK) # body body = record[8].decode('utf-8') if bool(record[8]) else record[8] + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) # location - location = [ f'ZCONVERSATIONMO (Z_PK: {record[0]})' ] - if record[1] is not None: location.append(f'ZCONVERSATIONSTATUSMO (Z_PK: {record[1]})') - if record[2] is not None: location.append(f'ZMESSAGEMO (Z_PK: {record[2]})') - location = ', '.join(location) + location = [ f"ZCONVERSATIONMO (Z_PK: {record[0]})" ] + if record[1] is not None: location.append(f"ZCONVERSATIONSTATUSMO (Z_PK: {record[1]})") + if record[2] is not None: location.append(f"ZMESSAGEMO (Z_PK: {record[2]})") + location = COMMA_SEP.join(location) # html row data_list_html.append((created, owner, record[5], admins_html, participants_html, body, record[9], last_updated, - record[11], record[12], location)) - + record[11], record[12], source_file_name_html, location)) # lava row data_list.append((created, owner, record[5], admins, participants, body, record[9], last_updated, - record[11], record[12], location)) - - # lava types - data_headers[0] = (data_headers[0], 'datetime') - data_headers[10] = (data_headers[10], 'datetime') + record[11], record[12], source_file_name, location)) - return data_headers, (data_list, data_list_html), source_path + return data_headers, (data_list, data_list_html), ' ' From 8ec0c6766e5262fa48886a46752bbf29b8e12034 Mon Sep 17 00:00:00 2001 From: Django Faiola <157513033+djangofaiola@users.noreply.github.com> Date: Sun, 4 May 2025 12:02:20 +0200 Subject: [PATCH 2/4] Update waze.py for lava output --- scripts/artifacts/waze.py | 1175 ++++++++++++++++++++----------------- 1 file changed, 622 insertions(+), 553 deletions(-) diff --git a/scripts/artifacts/waze.py b/scripts/artifacts/waze.py index 4193231a..ece87f28 100644 --- a/scripts/artifacts/waze.py +++ b/scripts/artifacts/waze.py @@ -1,613 +1,682 @@ __artifacts_v2__ = { - "waze": { - "name": "Waze", - "description": "Get account, session, searched locations, recent locations, favorite locations, " - "share locations, text-to-speech navigation and track GPS quality.", - "author": "Django Faiola (djangofaiola.blogspot.com @DjangoFaiola)", - "version": "0.1.2", - "date": "2024-02-02", + "waze_account": { + "name": "Account", + "description": "Parses and extract Waze Account", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", "requirements": "none", "category": "Waze", - "notes": "", - "paths": ('*/mobile/Containers/Data/Application/*/Documents/user.db*', - '*/mobile/Containers/Data/Application/*/.com.apple.mobile_container_manager.metadata.plist'), - "function": "get_waze" + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv" ], + "artifact_icon": "user" + }, + "waze_session_info": { + "name": "Session Info", + "description": "Parses and extract Waze Session Info", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "navigation-2" + }, + "waze_track_gps_quality": { + "name": "Track GPS Quality", + "description": "Parses and extract Waze Track GPS Quality", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "navigation-2" + }, + "waze_searched_locations": { + "name": "Searched Locations", + "description": "Parses and extract Waze Searched Locations", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "search" + }, + "waze_recent_locations": { + "name": "Recent Locations", + "description": "Parses and extract Waze Recent Locations", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "map-pin" + }, + "waze_favorite_locations": { + "name": "Favorite Locations", + "description": "Parses and extract Waze Favorite Locations", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "star" + }, + "waze_share_locations": { + "name": "Share Locations", + "description": "Parses and extract Waze Share Locations", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "map-pin" + }, + "waze_tts": { + "name": "Text-To-Speech navigation", + "description": "Parses and extract Waze Text-To-Speech navigation", + "author": "@djangofaiola", + "version": "0.2", + "creation_date": "2024-02-02", + "last_update_date": "2025-05-04", + "requirements": "none", + "category": "Waze", + "notes": "https://djangofaiola.blogspot.com", + "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), + "output_types": [ "lava", "html", "tsv", "timeline" ], + "artifact_icon": "volume-2" } } -import os import re -import plistlib -import pathlib -import shutil -import sqlite3 -import textwrap -import datetime - -from scripts.artifact_report import ArtifactHtmlReport -from scripts.ilapfuncs import logfunc, tsv, timeline, kmlgen, open_sqlite_db_readonly, convert_ts_int_to_utc, convert_utc_human_to_timezone - -# format location -def FormatLocation(location, value, tableName, key): - newLocation = '' - if value: - s = value.split(chr(29)) - for elem in range(0, len(s)): - if bool(s[elem]) and (s[elem].lower() != 'none'): - if newLocation: - newLocation = newLocation + ', ' - newLocation = newLocation + '(' + key + ': ' + s[elem] + ')' - if newLocation: - newLocation = tableName + ' ' + newLocation - if location: - newLocation = ', ' + newLocation - return location + newLocation - - -def FormatTimestamp(utc, timezone_offset): - if not bool(utc) or (utc == None): - return '' - else: - timestamp = convert_ts_int_to_utc(int(float(utc))) - return convert_utc_human_to_timezone(timestamp, timezone_offset) +from pathlib import Path +from scripts.ilapfuncs import get_file_path, get_sqlite_db_records, get_txt_file_content, convert_unix_ts_to_utc, artifact_processor, logfunc -# account -def get_account(file_found, report_folder, timezone_offset): - data_list = [] +# constants +LINE_BREAK = '\n' +COMMA_SEP = ', ' +HTML_LINE_BREAK = '
' - f = open(file_found, "r", encoding="utf-8") - try: - row = [ None ] * 5 - patternFirstName = 'Realtime.FirstName:' - patternLastName = 'Realtime.LastName:' - patternUserName = 'Realtime.Name:' - patternNickname = 'Realtime.Nickname:' - patternFirstLaunched = 'General.Last upgrade time:' - sep = ': ' - - data = f.readlines() - for line in data: - root = line.split('.', 1)[0] - if not root in ( 'Realtime', 'General' ): - continue - - # first name - if line.startswith(patternFirstName): - row[0] = line.split(sep, 1)[1] - # last name - elif line.startswith(patternLastName): - row[1] = line.split(sep, 1)[1] - # user name - elif line.startswith(patternUserName): - row[2] = line.split(sep, 1)[1] - # nickname - elif line.startswith(patternNickname): - row[3] = line.split(sep, 1)[1] - # first launched - elif line.startswith(patternFirstLaunched): - timestamp = line.split(sep, 1)[1] - row[4] = FormatTimestamp(timestamp, timezone_offset) - - # row - if row.count(None) != len(row): - data_list.append((row[0], row[1], row[2], row[3], row[4])) - - finally: - f.close() - - if len(data_list) > 0: - report = ArtifactHtmlReport('Waze Account') - report.start_artifact_report(report_folder, 'Waze Account') - report.add_script() - data_headers = ('First name', 'Last name', 'User name', 'Nickname', 'First launched') - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Account' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Account' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Account data available') - - -# session -def get_session(file_found, report_folder, timezone_offset): - data_list = [] - f = open(file_found, "r", encoding="utf-8") - try: - row = [ None ] * 8 - patternLastSynced = 'Config.Last synced:' - patternGPSPosition = 'GPS.Position:' - patternLastPosition = 'Navigation.Last position:' - patternLastDestName = 'Navigation.Last dest name:' - patternLastDestState = 'Navigation.Last dest state:' - patternLastDestCity = 'Navigation.Last dest city:' - patternLastDestStreet = 'Navigation.Last dest street:' - patternLastDestHouse = 'Navigation.Last dest number:' - sep = ': ' - - data = f.readlines() - for line in data: - root = line.split('.', 1)[0] - if not root in ( 'Config', 'GPS', 'Navigation' ): - continue - - # Last synced (ms) - if line.startswith(patternLastSynced): - timestamp = int(float(line.split(sep, 1)[1]) / 1000) - row[0] = FormatTimestamp(timestamp, timezone_offset) - # last position - elif line.startswith(patternGPSPosition): - coordinates = line.split(sep, 1)[1].split(',') # lon,lat - row[1] = f'{float(coordinates[1]) / 1000000},{float(coordinates[0]) / 1000000}' - # last navigation coordinates - elif line.startswith(patternLastPosition): - coordinates = line.split(sep, 1)[1].split(',') # lon,lat - row[2] = f'{float(coordinates[1]) / 1000000},{float(coordinates[0]) / 1000000}' - # last navigation destination - elif line.startswith(patternLastDestName): - row[3] = line.split(sep, 1)[1] - # state - elif line.startswith(patternLastDestState): - row[4] = line.split(sep, 1)[1] - # city - elif line.startswith(patternLastDestCity): - row[5] = line.split(sep, 1)[1] - # street - elif line.startswith(patternLastDestStreet): - row[6] = line.split(sep, 1)[1] - # house - elif line.startswith(patternLastDestHouse): - row[7] = line.split(sep, 1)[1] - - # row - if row.count(None) != len(row): - data_list.append((row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7])) - - finally: - f.close() - - if len(data_list) > 0: - report = ArtifactHtmlReport('Waze Session info') - report.start_artifact_report(report_folder, 'Waze Session info') - report.add_script() - data_headers = ('Last synced', 'Last position', 'Last navigation coordinates', 'Last navigation destination', 'State', 'City', 'Street', 'House') - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Session info' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Session info' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Session info data available') +# device path/local path +def get_device_file_path(file_path, seeker): + device_path = file_path + if bool(file_path): + file_info = seeker.file_infos.get(file_path) if file_path else None + # data folder: /path/to/report/data + if file_info: + source_path = file_info.source_path + # extraction folder: /path/to/directory + else: + source_path = file_path + if source_path.startswith('\\\\?\\'): source_path = source_path[4:] + source_path = Path(source_path).as_posix() -# recent locations -def get_recent_locations(file_found, report_folder, database, timezone_offset): - cursor = database.cursor() - cursor.execute(''' - SELECT - R.id, - P.id, - R.access_time, - R.name AS "name", - CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) || "," || CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "coordinates", - R.created_time - FROM RECENTS AS "R" - LEFT JOIN PLACES AS "P" ON (R.place_id = P.id) - ''') - - all_rows = cursor.fetchall() - usageentries = len(all_rows) - if usageentries > 0: - report = ArtifactHtmlReport('Waze Recent locations') - report.start_artifact_report(report_folder, 'Waze Recent locations') - report.add_script() - data_headers = ('Last access', 'Name', 'Coordinates', 'Created', 'Location') - data_list = [] - for row in all_rows: - # R.id - location = FormatLocation('', str(row[0]), 'RECENTS', 'id') - - # P.id - location = FormatLocation(location, str(row[1]), 'PLACES', 'id') - - # last access - lastAccess = FormatTimestamp(row[2], timezone_offset) - - # created - created = FormatTimestamp(row[5], timezone_offset) - - # row - data_list.append((lastAccess, row[3], row[4], created, location)) - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Recent locations' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Recent locations' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Recent locations data available') + index_private = source_path.find('/private/') + if index_private > 0: + device_path = source_path[index_private:] + else: + device_path = source_path + return device_path -# favorite locations -def get_favorite_locations(file_found, report_folder, database, timezone_offset): - cursor = database.cursor() - cursor.execute(''' - SELECT - F.id, - P.id, - F.access_time, - F.name AS "name", - CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) || "," || CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "coordinates", - F.created_time, - F.modified_time - FROM FAVORITES AS "F" - LEFT JOIN PLACES AS "P" ON (F.place_id = P.id) - ''') - - all_rows = cursor.fetchall() - usageentries = len(all_rows) - if usageentries > 0: - report = ArtifactHtmlReport('Waze Favorite locations') - report.start_artifact_report(report_folder, 'Waze Favorite locations') - report.add_script() - data_headers = ('Last access', 'Name', 'Coordinates', 'Created', 'Modified', 'Location') - data_list = [] - for row in all_rows: - # F.id - location = FormatLocation('', str(row[0]), 'FAVORITES', 'id') - - # P.id - location = FormatLocation(location, str(row[1]), 'PLACES', 'id') - - # last access - lastAccess = FormatTimestamp(row[2], timezone_offset) - - # created - created = FormatTimestamp(row[5], timezone_offset) - - # modified - modified = FormatTimestamp(row[6], timezone_offset) - - # row - data_list.append((lastAccess, row[3], row[4], created, modified, location)) - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Favorite locations' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Favorite locations' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Favorite locations data available') - - -# shared locations -def get_shared_locations(file_found, report_folder, database, timezone_offset): - cursor = database.cursor() - cursor.execute(''' - SELECT - SP.id, - P.id, - SP.share_time, - SP.name AS "name", - CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) || "," || CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "coordinates", - SP.created_time, - SP.modified_time, - SP.access_time - FROM SHARED_PLACES AS "SP" - LEFT JOIN PLACES AS "P" ON (SP.place_id = P.id) - ''') - - all_rows = cursor.fetchall() - usageentries = len(all_rows) - if usageentries > 0: - report = ArtifactHtmlReport('Waze Shared locations') - report.start_artifact_report(report_folder, 'Waze Shared locations') - report.add_script() - data_headers = ('Shared', 'Name', 'Coordinates', 'Created', 'Modified', 'Last access', 'Location') - data_list = [] - for row in all_rows: - # SP.id - location = FormatLocation('', str(row[0]), 'SHARED_PLACES', 'id') - - # P.id - location = FormatLocation(location, str(row[1]), 'PLACES', 'id') - - # shared - shared = FormatTimestamp(row[2], timezone_offset) - - # created - created = FormatTimestamp(row[5], timezone_offset) - - # modified - modified = FormatTimestamp(row[6], timezone_offset) - - # last access - lastAccess = FormatTimestamp(row[7], timezone_offset) - - # row - data_list.append((shared, row[3], row[4], created, modified, lastAccess, location)) - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Shared locations' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Shared locations' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Shared locations data available') +# unordered list +def unordered_list(values, html_format=False): + if not bool(values): + return None -# searched locations -def get_searched_locations(file_found, report_folder, database, timezone_offset): - cursor = database.cursor() - cursor.execute(''' - SELECT - P.id, - P.created_time, - P.name, - P.street, - P.house, - P.state, - P.city, - P.country, - CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) || "," || CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "coordinates" - FROM PLACES AS "P" - ''') - - all_rows = cursor.fetchall() - usageentries = len(all_rows) - if usageentries > 0: - report = ArtifactHtmlReport('Waze Searched locations') - report.start_artifact_report(report_folder, 'Waze Searched locations') - report.add_script() - data_headers = ('Created', 'Name', 'Street', 'House', 'State', 'City', 'Country', 'Coordinates', 'Location') - data_list = [] - for row in all_rows: - # P.id - location = FormatLocation('', str(row[0]), 'PLACES', 'id') - - # created - created = FormatTimestamp(row[1], timezone_offset) - - # row - data_list.append((created, row[2], row[3], row[4], row[5], row[6], row[7], row[8], location)) - - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Searched locations' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Searched locations' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Searched locations data available') - - -# text-to-speech navigation -def get_tts(file_found, report_folder, timezone_offset): - db = open_sqlite_db_readonly(file_found) - try: - # list tables - cursor = db.execute(f"SELECT name FROM sqlite_master WHERE type='table'") - all_tables = cursor.fetchall() - if len(all_tables) == 0: - logfunc('No Waze Text-To-Speech navigation data available') - return - - for table in all_tables: - table_name = table[0] - cursor = db.cursor() - cursor.execute(''' - SELECT - rowid, - update_time, - text - FROM {0} - '''.format(table_name)) - - all_rows = cursor.fetchall() - usageentries = len(all_rows) - if usageentries > 0: - report = ArtifactHtmlReport('Waze Text-To-Speech navigation') - report.start_artifact_report(report_folder, 'Waze Text-To-Speech navigation') - report.add_script() - data_headers = ('Timestamp', 'Text', 'Location') - data_list = [] - for row in all_rows: - # rowid - location = FormatLocation('', str(row[0]), table_name, 'rowid') + return HTML_LINE_BREAK.join(values) if html_format else LINE_BREAK.join(values) - # timestamp - timestamp = FormatTimestamp(row[1], timezone_offset) - # row - data_list.append((timestamp, row[2], location)) +# get application id +def get_application_id(files_found): + file_found = get_file_path(files_found, "com.waze.iphone.plist") + return Path(file_found).parents[2].name if bool(file_found) else None - report.write_artifact_data_table(data_headers, data_list, file_found) - report.end_artifact_report() - - tsvname = f'Waze Text-To-Speech navigation' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Text-To-Speech navigation' - timeline(report_folder, tlactivity, data_list, data_headers) - else: - logfunc('No Waze Text-To-Speech navigation data available') - finally: - db.close() - - -# track gps quality -def get_gps_quality(files_found, report_folder, timezone_offset): + +# account +@artifact_processor +def waze_account(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + 'First name', + 'Last name', + 'User name', + 'Nickname', + ('First launched', 'datetime') + ) data_list = [] - source_files = [] + SEP = ': ' + first_name = None + last_name = None + user_name = None + nickname = None + first_launched = None + + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/user", return_on_first_hit=True) + lines = get_txt_file_content(file_found) + for line in lines: + root = line.split('.', 1)[0] + if not root in ( 'Realtime', 'General' ): + continue - for file_found in files_found: - file_found = str(file_found) - file_name = pathlib.Path(file_found).name + # first name + if line.startswith('Realtime.FirstName:'): + first_name = line.split(SEP, 1)[1] + # last name + elif line.startswith('Realtime.LastName:'): + last_name = line.split(SEP, 1)[1] + # user name + elif line.startswith('Realtime.Name:'): + user_name = line.split(SEP, 1)[1] + # nickname + elif line.startswith('Realtime.Nickname:'): + nickname = line.split(SEP, 1)[1] + # first launched + elif line.startswith('General.First use:'): + timestamp = float(line.split(SEP, 1)[1]) + first_launched = convert_unix_ts_to_utc(timestamp) + + # lava row + data_list.append((first_name, last_name, user_name, nickname, first_launched)) + + return data_headers, data_list, file_found + + +# session info +@artifact_processor +def waze_session_info(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Last synced', 'datetime'), + 'Last position latitude', + 'Last position longitude', + 'Last navigation latitude', + 'Last navigation longitude', + 'Last navigation destination', + 'State', + 'City', + 'Street', + 'House' + ) + data_list = [] + SEP = ': ' + last_synced = None + latitude = None + longitude = None + last_latitude = None + last_longitude = None + last_dest_name = None + state = None + city = None + street = None + house = None + + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/session", return_on_first_hit=True) + lines = get_txt_file_content(file_found) + for line in lines: + root = line.split('.', 1)[0] + if not root in ( 'Config', 'GPS', 'Navigation' ): + continue + # Last synced (ms) + if line.startswith('Config.Last synced:'): + timestamp = float(line.split(SEP, 1)[1]) + last_synced = convert_unix_ts_to_utc(timestamp) + # last position + elif line.startswith('GPS.Position:'): + coordinates = line.split(SEP, 1)[1].split(',') # lon,lat + latitude = f"{float(coordinates[1]) / 1000000}" + longitude = f"{float(coordinates[0]) / 1000000}" + # last navigation coordinates + elif line.startswith('Navigation.Last position:'): + coordinates = line.split(SEP, 1)[1].split(',') # lon,lat + last_latitude = f"{float(coordinates[1]) / 1000000}" + last_longitude = f"{float(coordinates[0]) / 1000000}" + # last navigation destination + elif line.startswith('Navigation.Last dest name:'): + last_dest_name = line.split(SEP, 1)[1] + # state + elif line.startswith('Navigation.Last dest state:'): + state = line.split(SEP, 1)[1] + # city + elif line.startswith('Navigation.Last dest city:'): + city = line.split(SEP, 1)[1] + # street + elif line.startswith('Navigation.Last dest street:'): + street = line.split(SEP, 1)[1] + # house + elif line.startswith('Navigation.Last dest number:'): + house = line.split(SEP, 1)[1] + + # lava row + data_list.append((last_synced, latitude, longitude, last_latitude, last_longitude, last_dest_name, state, city, street, house)) + + return data_headers, data_list, file_found + + +# track gps tracker +@artifact_processor +def waze_track_gps_quality(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Timestamp', 'datetime'), + 'Latitude', + 'Longitude', + 'Sample count (bad)', + 'Average accuracy (min-max)', + 'Provider', + 'Source file name', + 'Location' + ) + data_list = [] + data_list_html = [] + device_file_paths = [] + artifact_info_name = __artifacts_v2__['waze_track_gps_quality']['name'] + + waze_app_identifier = get_application_id(files_found) + spdlog = seeker.search(f"*/{waze_app_identifier}/Documents/spdlog.*logdata") + + # all files + for file_found in spdlog: + file_name = Path(file_found).name + device_file_path = get_device_file_path(file_found, seeker) + + # spdlog.*logdata if not (file_name.startswith('spdlog') and file_name.endswith('.logdata')): continue - f = open(file_found, "r", encoding="utf-8") try: - row = [ None ] * 6 - hit_count = 0 - line_count = 0 - line_filter = re.compile(r'STAT\(buffer#[\d]{1,2}\)\sGPS_QUALITY\s') - values_filter = re.compile(r'(?<=\{)(.*?)(?=\})') + device_file_paths = [ device_file_path ] + + # regexpr pattern + line_pattern = re.compile(r'STAT\(buffer#[\d]{1,2}\)\sGPS_QUALITY\s') + values_pattern = re.compile(r'(?<=\{)(.*?)(?=\})') - data = f.readlines() - for line in data: + # text file + lines = get_txt_file_content(file_found) + + line_count = 0 + for line in lines: line_count += 1 - + + device_file_paths = [ device_file_path ] + # gps quality - if not re.search(line_filter, line): + if not re.search(line_pattern, line): continue - hit_count += 1 - location = FormatLocation('', str(line_count), file_name, 'row') - - values_iter = re.finditer(values_filter, line) + timestamp = None + latitude = None + longitude = None + sample_count = None + average_accuracy = None + provider = None + + values_iter = re.finditer(values_pattern, line) for kv in values_iter: kv_split = kv.group().split('=', 1) - + # timestamp if kv_split[0] == 'TIMESTAMP': - row[0] = FormatTimestamp(kv_split[1], timezone_offset) + timestamp = convert_unix_ts_to_utc(float(kv_split[1])) # latitude elif kv_split[0] == 'LAT': - row[1] = float(kv_split[1]) / 1000000 + latitude = float(kv_split[1]) / 1000000 # longitude elif kv_split[0] == 'LON': - row[2] = float(kv_split[1]) / 1000000 + longitude = float(kv_split[1]) / 1000000 # sample count elif kv_split[0] == 'SAMPLE_COUNT': - row[3] = kv_split[1] + sample_count = kv_split[1] # bad sample count elif kv_split[0] == 'BAD_SAMPLE_COUNT': - row[3] += ' (' + kv_split[1] + ')' + sample_count += f" ({kv_split[1]})" # accuracy "avg (min-max)" elif kv_split[0] == 'ACC_AVG': - row[4] = kv_split[1] + average_accuracy = kv_split[1] # accuracy "avg (min-max)" elif kv_split[0] == 'ACC_MIN': - row[4] += ' (' + kv_split[1] + '-' + average_accuracy += f" ({kv_split[1]}-" # accuracy "avg (min-max)" elif kv_split[0] == 'ACC_MAX': - row[4] += kv_split[1] + ')' + average_accuracy += f"{kv_split[1]})" # provider elif kv_split[0] == 'PROVIDER': - row[5] = kv_split[1] - - # row - if row.count(None) != len(row): - data_list.append((row[0], row[1], row[2], row[3], row[4], row[5], location)) - - if hit_count > 0: - if file_found.startswith('\\\\?\\'): - source_files.append(file_found[4:]) - else: - source_files.append(file_found) - finally: - f.close() - - if len(data_list) > 0: - report = ArtifactHtmlReport('Waze Track GPS quality') - report.start_artifact_report(report_folder, 'Waze Track GPS quality') - report.add_script() - data_headers = ('Timestamp', 'Latitude', 'Longitude', 'Sample count (bad)', 'Average accuracy (min-max)', 'Provider', 'Location') - - report.write_artifact_data_table(data_headers, data_list, ', '.join(source_files)) - report.end_artifact_report() - - tsvname = f'Waze Track GPS quality' - tsv(report_folder, data_headers, data_list, tsvname) - - tlactivity = f'Waze Track GPS quality' - timeline(report_folder, tlactivity, data_list, data_headers) - - kmlactivity = 'Waze Track GPS quality' - kmlgen(report_folder, kmlactivity, data_list, data_headers) - else: - logfunc('No Waze Track GPS quality data available') - - -# waze -def get_waze(files_found, report_folder, seeker, wrap_text, timezone_offset): - #datos = seeker.search('**/*com.apple.mobile_container_manager.metadata.plist') - for file_foundm in files_found: - if file_foundm.endswith('.com.apple.mobile_container_manager.metadata.plist'): - with open(file_foundm, 'rb') as f: - pl = plistlib.load(f) - if pl['MCMMetadataIdentifier'] == 'com.waze.iphone': - fulldir = (os.path.dirname(file_foundm)) - identifier = (os.path.basename(fulldir)) - - # user - path_list = seeker.search(f'*/{identifier}/Documents/user', True) - if len(path_list) > 0: - get_account(path_list, report_folder, timezone_offset) - - # session - path_list = seeker.search(f'*/{identifier}/Documents/session', True) - if len(path_list) > 0: - get_session(path_list, report_folder, timezone_offset) - - # tts.db - path_list = seeker.search(f'*/{identifier}/Library/Caches/tts/tts.db', True) - if len(path_list) > 0: - get_tts(path_list, report_folder, timezone_offset) - - # spdlog.*logdata - path_list = seeker.search(f'*/{identifier}/Documents/spdlog.*logdata') - if len(path_list) > 0: - get_gps_quality(path_list, report_folder, timezone_offset) - - break - - for file_found in files_found: - # user.db - if file_found.endswith('user.db'): - db = open_sqlite_db_readonly(file_found) - try: - # searched locations - get_searched_locations(file_found, report_folder, db, timezone_offset) - - # recent locations - get_recent_locations(file_found, report_folder, db, timezone_offset) - - # favorite locations - get_favorite_locations(file_found, report_folder, db, timezone_offset) - - # shared locations - get_shared_locations(file_found, report_folder, db, timezone_offset) - finally: - db.close() + provider = kv_split[1] + + # source file name + device_file_paths = dict.fromkeys(device_file_paths) + source_file_name = unordered_list(device_file_paths) + source_file_name_html = unordered_list(device_file_paths, html_format=True) + # location + location = f"{Path(file_found).name} (row: {line_count})" + + # html row + data_list_html.append((timestamp, latitude, longitude, sample_count, average_accuracy, provider, source_file_name_html, location)) + # lava row + data_list.append((timestamp, latitude, longitude, sample_count, average_accuracy, provider, source_file_name, location)) + except Exception as ex: + logfunc(f"Exception while parsing {artifact_info_name} - {file_found}: " + str(ex)) + pass + + return data_headers, (data_list, data_list_html), ' ' + + +# searched locations +@artifact_processor +def waze_searched_locations(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Created', 'datetime'), + 'Name', + 'Street', + 'House', + 'State', + 'City', + 'Country', + 'Latitude', + 'Longitude', + 'Location' + ) + data_list = [] + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/user.db", return_on_first_hit=True) + + query = ''' + SELECT + P.id, + P.created_time, + P.name, + P.street, + P.house, + P.state, + P.city, + P.country, + CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) AS "latitude", + CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "longitude" + FROM PLACES AS "P" + ''' + + db_records = get_sqlite_db_records(file_found, query) + for record in db_records: + # created + created = convert_unix_ts_to_utc(record[1]) + # name + name = record[2] + # street + street = record[3] + # house + house = record[4] + # state + state = record[5] + # city + city = record[6] + # country + country = record[7] + # latitude + latitude = record[8] + # longitude + longitude = record[9] + + # location + location = f"PLACES (id: {record[0]})" + + # lava row + data_list.append((created, name, street, house, state, city, country, latitude, longitude, location)) + + return data_headers, data_list, file_found + + +# recent locations +@artifact_processor +def waze_recent_locations(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Last access', 'datetime'), + 'Name', + 'Latitude', + 'Longitude', + ('Created', 'datetime'), + 'Location' + ) + data_list = [] + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/user.db", return_on_first_hit=True) + + query = ''' + SELECT + R.id, + P.id, + R.access_time, + R.name AS "name", + CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) AS "latitude", + CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "longitude", + R.created_time + FROM RECENTS AS "R" + LEFT JOIN PLACES AS "P" ON (R.place_id = P.id) + ''' + + db_records = get_sqlite_db_records(file_found, query) + for record in db_records: + # last access + last_access = convert_unix_ts_to_utc(record[2]) + # name + name = record[3] + # latitude + latitude = record[4] + # longitude + longitude = record[5] + # created + created = convert_unix_ts_to_utc(record[6]) + + # location + location = [ f"RECENTS (id: {record[0]})" ] + if record[1] is not None: location.append(f"PLACES (id: {record[1]})") + location = COMMA_SEP.join(location) + + # lava row + data_list.append((last_access, name, latitude, longitude, created, location)) + + return data_headers, data_list, file_found + + +# favorite locations +@artifact_processor +def waze_favorite_locations(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Last access', 'datetime'), + 'Name', + 'Latitude', + 'Longitude', + ('Created', 'datetime'), + ('Modified', 'datetime'), + 'Location' + ) + data_list = [] + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/user.db", return_on_first_hit=True) + + query = ''' + SELECT + F.id, + P.id, + F.access_time, + F.name AS "name", + CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) AS "latitude", + CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "longitude", + F.created_time, + F.modified_time + FROM FAVORITES AS "F" + LEFT JOIN PLACES AS "P" ON (F.place_id = P.id) + ''' + + db_records = get_sqlite_db_records(file_found, query) + for record in db_records: + # last access + last_access = convert_unix_ts_to_utc(record[2]) + # name + name = record[3] + # latitude + latitude = record[4] + # longitude + longitude = record[5] + # created + created = convert_unix_ts_to_utc(record[6]) + # modified + modified = convert_unix_ts_to_utc(record[7]) + + # location + location = [ f"FAVORITES (id: {record[0]})" ] + if record[1] is not None: location.append(f"PLACES (id: {record[1]})") + location = COMMA_SEP.join(location) + + # lava row + data_list.append((last_access, name, latitude, longitude, created, modified, location)) + + return data_headers, data_list, file_found + + +# share locations +@artifact_processor +def waze_share_locations(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Shared', 'datetime'), + 'Name', + 'Latitude', + 'Longitude', + ('Created', 'datetime'), + ('Modified', 'datetime'), + ('Last access', 'datetime'), + 'Location' + ) + data_list = [] + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Documents/user.db", return_on_first_hit=True) + + query = ''' + SELECT + SP.id, + P.id, + SP.share_time, + SP.name AS "name", + CAST((CAST(P.latitude AS REAL) / 1000000) AS TEXT) AS "latitude", + CAST((CAST(P.longitude AS REAL) / 1000000) AS TEXT) AS "longitude", + SP.created_time, + SP.modified_time, + SP.access_time + FROM SHARED_PLACES AS "SP" + LEFT JOIN PLACES AS "P" ON (SP.place_id = P.id) + ''' + + db_records = get_sqlite_db_records(file_found, query) + for record in db_records: + # share time + shared = convert_unix_ts_to_utc(record[2]) + # name + name = record[3] + # latitude + latitude = record[4] + # longitude + longitude = record[5] + # created + created = convert_unix_ts_to_utc(record[6]) + # modified + modified = convert_unix_ts_to_utc(record[7]) + # last access + last_access = convert_unix_ts_to_utc(record[8]) + + # location + location = [ f"SHARED_PLACES (id: {record[0]})" ] + if record[1] is not None: location.append(f"PLACES (id: {record[1]})") + location = COMMA_SEP.join(location) + + # lava row + data_list.append((shared, name, latitude, longitude, created, modified, last_access, location)) + + return data_headers, data_list, file_found + + +# Text-To-Speech navigation +@artifact_processor +def waze_tts(files_found, report_folder, seeker, wrap_text, timezone_offset): + + data_headers = ( + ('Timestamp', 'datetime'), + 'Text', + 'Location' + ) + data_list = [] + waze_app_identifier = get_application_id(files_found) + file_found = seeker.search(f"*/{waze_app_identifier}/Library/Caches/tts/tts.db", return_on_first_hit=True) + + # list tables + query = f"SELECT name FROM sqlite_master WHERE type='table'" + + all_tables = get_sqlite_db_records(file_found, query) + for table in all_tables: + # table name + table_name = table[0] + + query = ''' + SELECT + rowid, + update_time, + text + FROM {0} + '''.format(table_name) + + db_records = get_sqlite_db_records(file_found, query) + for record in db_records: + # share timestamp + timestamp = convert_unix_ts_to_utc(record[1]) + # text + text = record[2] + + # location + location = f"{table_name} (rowid: {record[0]})" + + # lava row + data_list.append((timestamp, text, location)) + + return data_headers, data_list, file_found From 213b797b42846cdaf36dfc1c35529f3f2d890fb7 Mon Sep 17 00:00:00 2001 From: Django Faiola <157513033+djangofaiola@users.noreply.github.com> Date: Thu, 8 May 2025 04:41:28 +0200 Subject: [PATCH 3/4] Added kml output --- scripts/artifacts/BeReal.py | 7 +++++-- scripts/artifacts/waze.py | 12 ++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/artifacts/BeReal.py b/scripts/artifacts/BeReal.py index 87dadf2f..5eb0b7b4 100644 --- a/scripts/artifacts/BeReal.py +++ b/scripts/artifacts/BeReal.py @@ -116,7 +116,7 @@ '*/mobile/Containers/Shared/AppGroup/*/disk-bereal-Production_postFeedItems/*', '*/mobile/Containers/Data/Application/*/Library/Caches/AlexisBarreyat.BeReal/Cache.db*', '*/mobile/Containers/Shared/AppGroup/*/EntitiesStore.sqlite*'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "html_columns": [ "Primary URL", "Secondary URL", "Thumbnail URL", "Song URL", "Visibility", "Tagged friends", "Source file name", "Location" ], "artifact_icon": "calendar" }, @@ -757,12 +757,15 @@ def get_device_file_path(file_path, seeker): # extraction folder: /path/to/directory else: source_path = file_path + if source_path.startswith('\\\\?\\'): source_path = source_path[4:] source_path = Path(source_path).as_posix() index_private = source_path.find('/private/') if index_private > 0: device_path = source_path[index_private:] - + else: + device_path = source_path + return device_path diff --git a/scripts/artifacts/waze.py b/scripts/artifacts/waze.py index ece87f28..804f70d3 100644 --- a/scripts/artifacts/waze.py +++ b/scripts/artifacts/waze.py @@ -24,7 +24,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "lava", "html", "tsv", "timeline", "kml" ], "artifact_icon": "navigation-2" }, "waze_track_gps_quality": { @@ -38,7 +38,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "artifact_icon": "navigation-2" }, "waze_searched_locations": { @@ -52,7 +52,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "artifact_icon": "search" }, "waze_recent_locations": { @@ -66,7 +66,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "artifact_icon": "map-pin" }, "waze_favorite_locations": { @@ -80,7 +80,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "artifact_icon": "star" }, "waze_share_locations": { @@ -94,7 +94,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline" ], + "output_types": [ "all" ], "artifact_icon": "map-pin" }, "waze_tts": { From 92e095f60eec073203f16b1586ddf5825a59b0d1 Mon Sep 17 00:00:00 2001 From: Django Faiola <157513033+djangofaiola@users.noreply.github.com> Date: Thu, 8 May 2025 04:46:59 +0200 Subject: [PATCH 4/4] Add files via upload --- scripts/artifacts/waze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/artifacts/waze.py b/scripts/artifacts/waze.py index 804f70d3..0565eecf 100644 --- a/scripts/artifacts/waze.py +++ b/scripts/artifacts/waze.py @@ -24,7 +24,7 @@ "category": "Waze", "notes": "https://djangofaiola.blogspot.com", "paths": ('*/mobile/Containers/Data/Application/*/Preferences/com.waze.iphone.plist'), - "output_types": [ "lava", "html", "tsv", "timeline", "kml" ], + "output_types": [ "lava", "html", "tsv", "timeline" ], "artifact_icon": "navigation-2" }, "waze_track_gps_quality": {