diff --git a/app/src/main/java/com/wytehat/btlogger/NavigationHelper.java b/app/src/main/java/com/wytehat/btlogger/NavigationHelper.java index 1138564..cd94c53 100644 --- a/app/src/main/java/com/wytehat/btlogger/NavigationHelper.java +++ b/app/src/main/java/com/wytehat/btlogger/NavigationHelper.java @@ -51,2113 +51,7 @@ public final class NavigationHelper { Class target; if (page == 0) target = MainActivity.class; else if (page == 1) target = DeviceManagerActivity.class; - else if (page == 2) target = #!/usr/bin/env python3 - - """ - ====================================================================== - Memory Codec v11 - STRUCTURAL + SEMANTIC JSONL MEMORY RETRIEVAL BENCHMARK - - LOSSLESS AUTHORITATIVE STORAGE - LOCAL STRUCTURAL RETRIEVAL - QUERY-AWARE RERANKING - PROGRESSIVE DISCLOSURE - NO ENCODEC - NO LLM REQUIRED - ====================================================================== - - Default blind-test input: - - /Users/n0tst3v3/llm/.agent/conversations.jsonl - - Architecture: - - conversations.jsonl - | - +--> LOSSLESS AUTHORITATIVE STORAGE - | - +--> STRUCTURAL INDEX - | | - | +--> paths - | +--> filenames - | +--> tools - | +--> identifiers - | +--> terms - | +--> roles - | - +--> SEMANTIC CAPSULE - | - +--> LOCAL RETRIEVAL - | - +--> candidate generation - +--> exact structural matching - +--> weighted lexical matching - +--> identifier matching - +--> reranking - | - +--> minimal evidence - | - +--> LLM - """ - - from __future__ import annotations - - import argparse - import hashlib - import json - import math - import re - import time - import zlib - - from collections import Counter, defaultdict - from pathlib import Path - - - VERSION = "11" - - DEFAULT_INPUT = Path( - "/Users/n0tst3v3/llm/.agent/conversations.jsonl" - ) - - DEFAULT_OUTPUT = Path( - "/Users/n0tst3v3/llm/memory_codec_v11" - ) - - - # ---------------------------------------------------------------------- - # TOKEN / FEATURE EXTRACTION - # ---------------------------------------------------------------------- - - TOKEN_RE = re.compile( - r"[A-Za-z0-9_./:@+-]+" - ) - - PATH_RE = re.compile( - r"(?:/Users/|/storage/|/[^ \n`\"']+)" - ) - - FILE_RE = re.compile( - r"\b[\w.-]+\.(?:java|kt|py|json|jsonl|xml|gradle|md|txt|js|ts|sh|yaml|yml)\b" - ) - - TOOL_RE = re.compile( - r"\b([A-Za-z_][A-Za-z0-9_]*(?:_[A-Za-z0-9_]+)+)\s*\(" - ) - - CAMEL_RE = re.compile( - r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|\d+" - ) - - - STOP_WORDS = { - "the", - "and", - "for", - "with", - "that", - "this", - "what", - "does", - "about", - "from", - "into", - "have", - "your", - "you", - "are", - "was", - "were", - "how", - "why", - "when", - "where", - "which", - "conversation", - "say", - "says", - "tell", - "me", - "can", - "could", - "would", - "should", - "use", - "using", - "used", - "want", - "need", - "let", - "get", - "got", - "like", - "just", - "then", - "than", - "also", - "there", - "their", - "they", - "them", - "its", - "it's", - "our", - "we", - "i", - "a", - "an", - "of", - "to", - "in", - "on", - "is", - "it", - "as", - "at", - "or", - "be", - "do", - "did", - } - - - def tokens(text: str) -> list[str]: - return [ - x.lower() - for x in TOKEN_RE.findall(text) - ] - - - def meaningful(text: str) -> list[str]: - - result = [] - - for token in tokens(text): - - if len(token) <= 1: - continue - - if token in STOP_WORDS: - continue - - result.append(token) - - # Try to expose semantic pieces from CamelCase names. - # - # BluetoothTrackingService - # - # becomes roughly: - # - # bluetooth - # tracking - # service - - if any(c.isupper() for c in token): - - pieces = CAMEL_RE.findall(token) - - for piece in pieces: - - piece = piece.lower() - - if ( - len(piece) > 1 - and piece not in STOP_WORDS - ): - result.append(piece) - - return result - - - def estimate_tokens(text: str) -> int: - - # This is intentionally only an estimate. - # - # v11 does NOT require an LLM tokenizer. - - return max( - 1, - math.ceil(len(text) / 4) - ) - - - def sha256_bytes(data: bytes) -> str: - - return hashlib.sha256(data).hexdigest() - - - # ---------------------------------------------------------------------- - # FEATURE EXTRACTION - # ---------------------------------------------------------------------- - - def extract_features(text: str) -> dict: - - paths = [] - - for match in PATH_RE.findall(text): - - path = match.rstrip( - ".,;:)]}" - ) - - if len(path) > 3: - paths.append(path) - - files = FILE_RE.findall(text) - - tools = TOOL_RE.findall(text) - - identifiers = set() - - for item in files: - identifiers.add( - item.lower() - ) - - for item in tools: - identifiers.add( - item.lower() - ) - - command_terms = [] - - for line in text.splitlines(): - - stripped = line.strip() - - if ( - stripped.startswith("") - or stripped.startswith("```") - or stripped.startswith("$ ") - ): - - command_terms.extend( - meaningful(stripped) - ) - - return { - "terms": sorted( - set(meaningful(text)) - ), - - "paths": sorted( - set(paths) - ), - - "files": sorted( - set( - x.lower() - for x in files - ) - ), - - "tools": sorted( - set( - x.lower() - for x in tools - ) - ), - - "identifiers": sorted( - identifiers - ), - - "command_terms": sorted( - set(command_terms) - ), - - "roles": [], - - "raw_length": len(text), - } - - - # ---------------------------------------------------------------------- - # JSONL LOADING - # ---------------------------------------------------------------------- - - def load_jsonl( - path: Path, - ) -> tuple[list[dict], bytes]: - - raw = path.read_bytes() - - records = [] - - for line_number, line in enumerate( - raw.splitlines(), - 1, - ): - - if not line.strip(): - continue - - try: - - obj = json.loads(line) - - obj["_line"] = line_number - - records.append(obj) - - except json.JSONDecodeError as exc: - - raise SystemExit( - f"Invalid JSONL at line " - f"{line_number}: {exc}" - ) - - return records, raw - - - def message_text(obj: dict) -> str: - - content = obj.get( - "content", - "", - ) - - if isinstance( - content, - str, - ): - - return content - - return json.dumps( - content, - ensure_ascii=False, - sort_keys=True, - ) - - - # ---------------------------------------------------------------------- - # CHUNKING - # ---------------------------------------------------------------------- - - def build_chunks( - records: list[dict], - target_messages: int = 8, - ) -> list[dict]: - - chunks = [] - - for start in range( - 0, - len(records), - target_messages, - ): - - group = records[ - start: - start + target_messages - ] - - parts = [] - - for record in group: - - role = record.get( - "role", - "unknown", - ) - - content = message_text( - record - ) - - parts.append( - f"{role}: {content}" - ) - - text = "\n".join(parts) - - features = extract_features( - text - ) - - roles = sorted( - set( - str( - r.get( - "role", - "unknown", - ) - ) - for r in group - ) - ) - - features["roles"] = roles - - chunks.append( - { - "id": - f"chunk_{len(chunks):05d}", - - "record_start": - start, - - "record_end": - start + len(group) - 1, - - "line_start": - group[0]["_line"], - - "line_end": - group[-1]["_line"], - - "message_count": - len(group), - - "roles": - roles, - - "text": - text, - - "features": - features, - } - ) - - return chunks - - - # ---------------------------------------------------------------------- - # STRUCTURAL INDEX - # ---------------------------------------------------------------------- - - def build_structural_index(chunks): - """ - Build the local structural retrieval index. - - v11: - - Never assumes a fixed set of index fields. - - Dynamically creates buckets as fields are encountered. - - Supports terms, entities, paths, tools, roles, files, - commands, symbols, and future structural metadata. - - Everything remains local and is NOT sent to the LLM. - """ - - indexes = {} - - def normalize_value(value): - if value is None: - return [] - - if isinstance(value, str): - value = value.strip() - return [value] if value else [] - - if isinstance(value, (list, tuple, set)): - result = [] - - for item in value: - if item is None: - continue - - if isinstance(item, (str, int, float, bool)): - text = str(item).strip() - - if text: - result.append(text) - - return result - - if isinstance(value, (int, float, bool)): - return [str(value)] - - return [] - - def add_index(field, value, chunk_id): - if field not in indexes: - indexes[field] = {} - - if value not in indexes[field]: - indexes[field][value] = [] - - if chunk_id not in indexes[field][value]: - indexes[field][value].append(chunk_id) - - for chunk_number, chunk in enumerate(chunks): - if not isinstance(chunk, dict): - continue - - chunk_id = chunk.get( - "id", - f"chunk_{chunk_number:05d}" - ) - - for key, value in chunk.items(): - - # Ignore large/raw fields. - if key in { - "text", - "content", - "messages", - "raw", - "source" - }: - continue - - values = normalize_value(value) - - for normalized in values: - add_index( - key, - normalized, - chunk_id - ) - - return indexes - - - # ---------------------------------------------------------------------- - # QUERY FEATURE EXTRACTION - # ---------------------------------------------------------------------- - - def query_features( - query: str, - ) -> dict: - - features = extract_features( - query - ) - - query_terms = set( - features["terms"] - ) - - # Natural-language aliases. - # - # This lets: - # - # "repository" - # - # overlap with: - # - # "repo" - # - # without requiring an embedding model. - - aliases = { - - "bluetoothlogger": - ["bluetoothlogger"], - - "bluetooth": - ["bluetooth"], - - "android": - ["android"], - - "gitea": - ["gitea"], - - "repository": - ["repository", "repo"], - - "repo": - ["repository", "repo"], - - "rollback": - ["rollback", "roll", "back"], - - "rolled": - ["rollback"], - - "plugins": - ["plugin", "plugins"], - - "plugin": - ["plugin", "plugins"], - - "tool": - ["tool"], - - "runner": - ["runner"], - - "results": - ["results", "result"], - - "result": - ["results", "result"], - - "indexing": - ["indexing", "index"], - - "retrieved": - ["retrieved", "retrieve", "retrieval"], - - "memory": - ["memory", "memories"], - - "memories": - ["memory", "memories"], - - "disclosure": - ["disclosure"], - - "storage": - ["storage"], - - "authoritative": - ["authoritative"], - - "semantic": - ["semantic"], - - "lossy": - ["lossy"], - - "project": - ["project"], - - "path": - ["path"], - - "file": - ["file", "files"], - - "files": - ["file", "files"], - - "command": - ["command"], - - "commands": - ["command"], - - } - - expanded = set( - query_terms - ) - - for term in query_terms: - - expanded.update( - aliases.get( - term, - [], - ) - ) - - features[ - "expanded_terms" - ] = sorted(expanded) - - return features - - - # ---------------------------------------------------------------------- - # IDF - # ---------------------------------------------------------------------- - - def idf_weights( - chunks: list[dict], - ) -> dict[str, float]: - - document_frequency = Counter() - - for chunk in chunks: - - document_frequency.update( - set( - chunk["features"]["terms"] - ) - ) - - document_count = len( - chunks - ) - - weights = {} - - for term, frequency in document_frequency.items(): - - weights[term] = ( - math.log( - (document_count + 1) - / - (frequency + 1) - ) - + 1.0 - ) - - return weights - - - def overlap_score( - query_terms: set[str], - values: list[str], - weights: dict[str, float], - ) -> float: - - return sum( - weights.get( - value, - 1.0, - ) - for value in query_terms.intersection( - values - ) - ) - - - # ---------------------------------------------------------------------- - # QUERY-AWARE RERANKER - # ---------------------------------------------------------------------- - - def rerank( - query: str, - chunks: list[dict], - top_k: int = 3, - ) -> list[tuple[dict, float, list[str]]]: - - query_data = query_features( - query - ) - - query_terms = set( - query_data[ - "expanded_terms" - ] - ) - - query_paths = set( - query_data["paths"] - ) - - query_files = set( - query_data["files"] - ) - - query_tools = set( - query_data["tools"] - ) - - weights = idf_weights( - chunks - ) - - scored = [] - - for chunk in chunks: - - features = chunk[ - "features" - ] - - matched = [] - - score = 0.0 - - # -------------------------------------------------------------- - # EXACT PATH MATCH - # -------------------------------------------------------------- - - exact_paths = ( - query_paths.intersection( - features["paths"] - ) - ) - - if exact_paths: - - score += ( - 14.0 - * - len(exact_paths) - ) - - matched.extend( - f"path:{x}" - for x in exact_paths - ) - - # -------------------------------------------------------------- - # EXACT FILE MATCH - # -------------------------------------------------------------- - - exact_files = ( - query_files.intersection( - features["files"] - ) - ) - - if exact_files: - - score += ( - 12.0 - * - len(exact_files) - ) - - matched.extend( - f"file:{x}" - for x in exact_files - ) - - # -------------------------------------------------------------- - # EXACT TOOL MATCH - # -------------------------------------------------------------- - - exact_tools = ( - query_tools.intersection( - features["tools"] - ) - ) - - if exact_tools: - - score += ( - 11.0 - * - len(exact_tools) - ) - - matched.extend( - f"tool:{x}" - for x in exact_tools - ) - - # -------------------------------------------------------------- - # WEIGHTED NORMAL TERMS - # -------------------------------------------------------------- - - term_hits = ( - query_terms.intersection( - features["terms"] - ) - ) - - if term_hits: - - lexical_score = overlap_score( - query_terms, - features["terms"], - weights, - ) - - score += ( - 2.2 - * - lexical_score - ) - - matched.extend( - sorted(term_hits) - ) - - # -------------------------------------------------------------- - # IDENTIFIER MATCH - # -------------------------------------------------------------- - - identifier_hits = ( - query_terms.intersection( - set( - features[ - "identifiers" - ] - ) - ) - ) - - if identifier_hits: - - score += ( - 4.0 - * - len(identifier_hits) - ) - - # -------------------------------------------------------------- - # COMMAND-AWARE BOOST - # -------------------------------------------------------------- - - if any( - x in query_terms - for x in ( - "command", - "commands", - "sshpass", - "read_file", - ) - ): - - if ( - features["command_terms"] - or - features["tools"] - ): - - score += 2.5 - - # -------------------------------------------------------------- - # MULTI-SIGNAL BOOST - # -------------------------------------------------------------- - - independent_signals = sum( - bool(x) - for x in ( - exact_paths, - exact_files, - exact_tools, - term_hits, - ) - ) - - if independent_signals >= 2: - - score += 3.0 - - if score > 0: - - scored.append( - ( - chunk, - score, - matched, - ) - ) - - # Highest score first. - # - # For equal scores, preserve conversation order. - - scored.sort( - key=lambda item: ( - -item[1], - item[0]["record_start"], - ) - ) - - return scored[:top_k] - - - # ---------------------------------------------------------------------- - # PROGRESSIVE EVIDENCE - # ---------------------------------------------------------------------- - - def make_evidence( - results, - max_chars: int = 2600, - ) -> str: - - pieces = [] - - used = 0 - - for chunk, score, matched in results: - - text = chunk["text"] - - # Prevent a giant assistant/code response - # from dominating the evidence packet. - - if len(text) > 1400: - - text = ( - text[:1400] - + - "\n[…truncated locally…]" - ) - - block = ( - f"[{chunk['id']} " - f"score={score:.2f}]\n" - f"{text}\n" - ) - - if ( - used - + - len(block) - > - max_chars - ): - - remaining = ( - max_chars - - - used - ) - - if remaining > 120: - - pieces.append( - block[:remaining] - ) - - break - - pieces.append( - block - ) - - used += len(block) - - return "\n".join( - pieces - ) - - - # ---------------------------------------------------------------------- - # SEMANTIC CAPSULE - # ---------------------------------------------------------------------- - - def make_capsule( - chunks: list[dict], - ) -> dict: - - facts = [] - procedures = [] - constraints = [] - entities = [] - - seen = set() - - for chunk in chunks: - - text = chunk["text"] - - lower = text.lower() - - # -------------------------------------------------------------- - # FACTS - # -------------------------------------------------------------- - - for line in text.splitlines(): - - stripped = line.strip() - - if not stripped: - continue - - if any( - phrase in lower - for phrase in ( - "decided", - "decision", - "we will", - "use gitea", - "selected repository", - ) - ): - - key = ( - "fact", - stripped, - ) - - if key not in seen: - - facts.append( - { - "chunk": - chunk["id"], - - "text": - stripped, - } - ) - - seen.add(key) - - # -------------------------------------------------------------- - # PROCEDURES - # -------------------------------------------------------------- - - if any( - phrase in lower - for phrase in ( - "how to", - "let me", - "command", - "read_file(", - "project_set(", - "gitea_", - "android_", - ) - ): - - first_line = next( - ( - line.strip() - for line - in text.splitlines() - if line.strip() - ), - "", - ) - - procedures.append( - { - "chunk": - chunk["id"], - - "summary": - first_line, - } - ) - - # -------------------------------------------------------------- - # CONSTRAINTS - # -------------------------------------------------------------- - - if any( - phrase in lower - for phrase in ( - "must", - "need to", - "supposed to", - "make sure", - "until", - ) - ): - - constraints.append( - { - "chunk": - chunk["id"], - - "summary": - text[:300], - } - ) - - # -------------------------------------------------------------- - # ENTITIES - # -------------------------------------------------------------- - - for entity in ( - chunk["features"]["paths"] - + - chunk["features"]["files"] - + - chunk["features"]["tools"] - ): - - if entity not in entities: - - entities.append( - entity - ) - - return { - - "version": - VERSION, - - "type": - "semantic_capsule", - - "authoritative_source": - "conversations.jsonl", - - "facts": - facts[:200], - - "procedures": - procedures[:200], - - "constraints": - constraints[:100], - - "entities": - entities[:500], - - "note": - ( - "Derived memory only; " - "authoritative content remains " - "lossless JSONL." - ), - } - - - # ---------------------------------------------------------------------- - # BLIND TESTS - # - # These are intentionally phrased more like actual future memory - # questions instead of copying the vocabulary directly from chunks. - # ---------------------------------------------------------------------- - - BLIND_TESTS = [ - - ( - "Where was the Android BluetoothLogger project located?", - "chunk_00000", - ), - - ( - "What was the assistant trying to inspect to understand " - "the Bluetooth/GPS application?", - "chunk_00001", - ), - - ( - "Which source file was being read under the " - "BluetoothLogger Java package?", - "chunk_00002", - ), - - ( - "Which Gitea repository was selected for the Android project?", - "chunk_00003", - ), - - ( - "What command was used to search the user's home directory " - "for Android files?", - "chunk_00004", - ), - - ( - "Why did the Android tools say that no project was detected?", - "chunk_00005", - ), - - ( - "What problem did the Android tools have with the project directory?", - "chunk_00006", - ), - - ( - "What was the TrackingService supposed to do when a relevant " - "broadcast action occurred?", - "chunk_00011", - ), - - ( - "Why was the user frustrated about which directory " - "the assistant was using?", - "chunk_00012", - ), - - ( - "What repository-selection approach did the assistant try?", - "chunk_00014", - ), - ] - - - # ---------------------------------------------------------------------- - # BLIND TEST RUNNER - # ---------------------------------------------------------------------- - - def run_blind_test( - chunks: list[dict], - ) -> list[dict]: - - rows = [] - - for test_number, ( - query, - expected, - ) in enumerate( - BLIND_TESTS, - 1, - ): - - start = time.perf_counter() - - results = rerank( - query, - chunks, - top_k=3, - ) - - elapsed_ms = ( - time.perf_counter() - - - start - ) * 1000 - - retrieved = [ - item[0]["id"] - for item in results - ] - - hit = ( - expected - in - retrieved - ) - - evidence = make_evidence( - results - ) - - rows.append( - { - "test": - test_number, - - "query": - query, - - "expected": - expected, - - "retrieved": - retrieved, - - "hit": - hit, - - "retrieval_ms": - round( - elapsed_ms, - 3, - ), - - "recall_tokens": - sum( - estimate_tokens( - item[0]["text"] - ) - for item in results - ), - - "evidence_tokens": - estimate_tokens( - evidence - ), - - "matched": - [ - item[2] - for item in results - ], - } - ) - - return rows - - - # ---------------------------------------------------------------------- - # REPORT - # ---------------------------------------------------------------------- - - def print_report( - source, - raw, - records, - chunks, - capsule, - compressed, - tests, - ): - - original_tokens = estimate_tokens( - raw.decode( - "utf-8", - "replace", - ) - ) - - average_recall = ( - sum( - x["recall_tokens"] - for x in tests - ) - / - len(tests) - ) - - average_evidence = ( - sum( - x["evidence_tokens"] - for x in tests - ) - / - len(tests) - ) - - hit_rate = ( - sum( - x["hit"] - for x in tests - ) - / - len(tests) - ) - - scan_tokens = sum( - estimate_tokens( - json.dumps( - chunk["features"], - ensure_ascii=False, - ) - ) - for chunk in chunks - ) - - capsule_json = json.dumps( - capsule, - ensure_ascii=False, - indent=2, - ).encode() - - print("=" * 70) - - print( - "Memory Codec v11 " - "STRUCTURAL + SEMANTIC JSONL MEMORY BENCHMARK" - ) - - print( - "LOSSLESS | LOCAL RETRIEVAL | RERANKING | " - "PROGRESSIVE DISCLOSURE" - ) - - print( - "NO ENCODEC | NO LLM" - ) - - print("=" * 70) - - print() - - print( - f"Input: {source}" - ) - - print( - f"Output: {DEFAULT_OUTPUT}" - ) - - print() - - print("-" * 70) - print("SOURCE") - print("-" * 70) - - print( - f"Original bytes: {len(raw)}" - ) - - print( - f"JSONL records: {len(records)}" - ) - - print( - "Messages: " - f"{sum(1 for r in records if r.get('type') == 'message')}" - ) - - print( - f"Estimated LLM tokens: {original_tokens}" - ) - - print( - f"SHA-256: " - f"{sha256_bytes(raw)}" - ) - - print() - - print("-" * 70) - print("LOSSLESS AUTHORITATIVE STORAGE") - print("-" * 70) - - print( - f"Compressed bytes: {len(compressed)}" - ) - - print( - "Storage ratio: " - f"{len(compressed) / len(raw):.4f}x" - ) - - print( - "Compression saved: " - f"{len(raw) - len(compressed)} bytes" - ) - - print( - "Exact recovery: PASS" - ) - - print() - - print("-" * 70) - print("LOCAL STRUCTURAL MEMORY") - print("-" * 70) - - print( - f"Chunks: {len(chunks)}" - ) - - print( - "Index entries: " - f"{sum(len(c['features']['terms']) for c in chunks)}" - ) - - print( - f"Local scan context: {scan_tokens} estimated tokens" - ) - - print() - - print( - "Indexed structures:" - ) - - print( - " paths" - ) - - print( - " filenames" - ) - - print( - " tools" - ) - - print( - " identifiers" - ) - - print( - " terms" - ) - - print( - " roles" - ) - - print() - - print("-" * 70) - print("SEMANTIC CAPSULE") - print("-" * 70) - - print( - f"Capsule JSON bytes: {len(capsule_json)}" - ) - - print( - "Capsule tokens: " - f"{estimate_tokens(capsule_json.decode())}" - ) - - print( - f"Facts: {len(capsule['facts'])}" - ) - - print( - f"Procedures: " - f"{len(capsule['procedures'])}" - ) - - print( - f"Constraints: " - f"{len(capsule['constraints'])}" - ) - - print( - f"Entities: " - f"{len(capsule['entities'])}" - ) - - print() - - print("-" * 70) - print("BLIND RETRIEVAL TEST") - print("-" * 70) - - for result in tests: - - print() - - print( - f"Test {result['test']}: " - f"{result['query']}" - ) - - print( - " Retrieval: " - f"{result['retrieval_ms']:.3f} ms" - ) - - print( - " Expected chunk: " - f"{result['expected']}" - ) - - print( - " Retrieved: " - f"{result['retrieved']}" - ) - - print( - " Recall tokens: " - f"{result['recall_tokens']}" - ) - - print( - " Evidence tokens: " - f"{result['evidence_tokens']}" - ) - - print( - " Expected hit: " - f"{'PASS' if result['hit'] else 'FAIL'}" - ) - - print() - - print("-" * 70) - print("LLM CONTEXT ACCOUNTING") - print("-" * 70) - - print( - f"Original LLM tokens: " - f"{original_tokens}" - ) - - print( - "Local scan LLM tokens: 0" - ) - - print( - f"Average recall tokens: " - f"{average_recall:.1f}" - ) - - print( - f"Average evidence tokens: " - f"{average_evidence:.1f}" - ) - - print( - "Recall reduction: " - f"{(1 - average_recall / original_tokens) * 100:.1f}%" - ) - - print( - "Evidence reduction: " - f"{(1 - average_evidence / original_tokens) * 100:.1f}%" - ) - - print( - f"Retrieval hit rate: " - f"{hit_rate * 100:.1f}%" - ) - - print() - - print("-" * 70) - print("V11 INTERPRETATION") - print("-" * 70) - - print( - "Retrieval quality: ", - end="", - ) - - if hit_rate >= 0.90: - - print( - "EXCELLENT" - ) - - elif hit_rate >= 0.80: - - print( - "GOOD" - ) - - else: - - print( - "NEEDS WORK" - ) - - print( - "Context reduction: ", - end="", - ) - - if ( - average_evidence - / - original_tokens - <= - 0.10 - ): - - print( - "EXCELLENT" - ) - - elif ( - average_evidence - / - original_tokens - <= - 0.20 - ): - - print( - "GOOD" - ) - - else: - - print( - "NEEDS WORK" - ) - - print() - - print("-" * 70) - print("FINAL RESULTS") - print("-" * 70) - - print( - f"Original JSONL bytes: " - f"{len(raw)}" - ) - - print( - f"Compressed authoritative: " - f"{len(compressed)}" - ) - - print( - f"Original estimated tokens: " - f"{original_tokens}" - ) - - print( - f"Average recall tokens: " - f"{average_recall:.1f}" - ) - - print( - f"Average evidence tokens: " - f"{average_evidence:.1f}" - ) - - print( - "Recall token reduction: " - f"{(1 - average_recall / original_tokens) * 100:.1f}%" - ) - - print( - "Evidence reduction: " - f"{(1 - average_evidence / original_tokens) * 100:.1f}%" - ) - - print( - f"Retrieval hit rate: " - f"{hit_rate * 100:.1f}%" - ) - - print() - - print("=" * 70) - - print( - "MEMORY CODEC v11 COMPLETE" - ) - - print("=" * 70) - - - # ---------------------------------------------------------------------- - # MAIN - # ---------------------------------------------------------------------- - - def main(): - - parser = argparse.ArgumentParser() - - parser.add_argument( - "input", - nargs="?", - type=Path, - default=DEFAULT_INPUT, - ) - - parser.add_argument( - "--output", - type=Path, - default=DEFAULT_OUTPUT, - ) - - args = parser.parse_args() - - source = ( - args.input - .expanduser() - .resolve() - ) - - output = ( - args.output - .expanduser() - .resolve() - ) - - output.mkdir( - parents=True, - exist_ok=True, - ) - - # -------------------------------------------------------------- - # LOAD SOURCE - # -------------------------------------------------------------- - - records, raw = load_jsonl( - source - ) - - # -------------------------------------------------------------- - # BUILD MEMORY - # -------------------------------------------------------------- - - chunks = build_chunks( - records - ) - - # -------------------------------------------------------------- - # LOSSLESS COMPRESSION - # -------------------------------------------------------------- - - compressed = zlib.compress( - raw, - level=9, - ) - - recovered = zlib.decompress( - compressed - ) - - if recovered != raw: - - raise SystemExit( - "FATAL: lossless recovery failed" - ) - - # -------------------------------------------------------------- - # STRUCTURAL INDEX - # -------------------------------------------------------------- - - structural_index = ( - build_structural_index( - chunks - ) - ) - - # -------------------------------------------------------------- - # SEMANTIC CAPSULE - # -------------------------------------------------------------- - - capsule = make_capsule( - chunks - ) - - capsule_json = json.dumps( - capsule, - ensure_ascii=False, - indent=2, - ).encode() - - capsule_compressed = zlib.compress( - capsule_json, - level=9, - ) - - # -------------------------------------------------------------- - # BLIND TEST - # -------------------------------------------------------------- - - tests = run_blind_test( - chunks - ) - - # -------------------------------------------------------------- - # MANIFEST - # -------------------------------------------------------------- - - manifest = { - - "version": - VERSION, - - "source": - str(source), - - "source_bytes": - len(raw), - - "source_sha256": - sha256_bytes(raw), - - "compressed_bytes": - len(compressed), - - "exact_recovery": - True, - - "no_encodec": - True, - - "no_llm": - True, - - "chunks": - len(chunks), - - "blind_tests": - len(tests), - - "architecture": [ - - "lossless_authoritative_storage", - - "structural_index", - - "semantic_capsule", - - "local_candidate_generation", - - "query_aware_reranking", - - "progressive_evidence", - - ], - } - - # -------------------------------------------------------------- - # WRITE ARTIFACTS - # -------------------------------------------------------------- - - ( - output - / - "conversations.jsonl.zlib" - ).write_bytes( - compressed - ) - - ( - output - / - "semantic_capsule.json" - ).write_bytes( - capsule_json - ) - - ( - output - / - "semantic_capsule.json.zlib" - ).write_bytes( - capsule_compressed - ) - - ( - output - / - "structural_index.json" - ).write_text( - json.dumps( - structural_index, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - - # Compact local index. - ( - output - / - "local_index.json" - ).write_text( - json.dumps( - [ - { - "id": - chunk["id"], - - "record_start": - chunk["record_start"], - - "record_end": - chunk["record_end"], - - "line_start": - chunk["line_start"], - - "line_end": - chunk["line_end"], - - "features": - chunk["features"], - } - - for chunk in chunks - ], - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - - ( - output - / - "retrieval_report.json" - ).write_text( - json.dumps( - tests, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - - ( - output - / - "manifest.json" - ).write_text( - json.dumps( - manifest, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - - # -------------------------------------------------------------- - # REPORT - # -------------------------------------------------------------- - - print_report( - source, - raw, - records, - chunks, - capsule, - compressed, - tests, - ) - - - if __name__ == "__main__": - - main().class; + else if (page == 2) target = MapActivity.class; else { Toast.makeText(activity, page < 0 ? "This is the first view" : "This is the last view", Toast.LENGTH_SHORT).show();