From 308d63b8d838ffff4b21c79a3647fd6c8d2680c6 Mon Sep 17 00:00:00 2001 From: n0tst3v3 Date: Wed, 19 Aug 2026 11:26:29 -0600 Subject: [PATCH] feat(map): upload recorded sightings to the ingestion server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ServerUploader plus an "Upload to Server" button and a Server… settings dialog in the map filter panel. The uploader builds a CSV of every device fix and every recorded history point - the history rows are the interesting ones, since they carry each device's location trail - and POSTs it as multipart/form-data with the API key in X-API-Key. Timestamps are written ISO-8601 in UTC, because sending local time would shift every point by the phone's offset. It uploads the whole database every time rather than tracking what was sent before. The server deduplicates on (mac, timestamp, latitude, longitude), so re-sending is idempotent: anything it already holds comes back counted as a duplicate instead of stored twice. Keeping the client dumb is what makes a failed or partial upload safe to simply retry, with no sync state to get out of step. Network work runs on a background thread and the result is delivered on the main thread; the button disables itself while a run is in flight and the panel shows the returned metrics. The URL and key default to the deployment at bt.justbug.me and are editable in the dialog, stored in the server_settings preferences. Verified against the real server: the exact CSV this emits - including empty RSSI cells for history rows and a device name containing a comma - ingests as 4 points, and re-uploading the same file returns 4 duplicates and 0 added. Co-Authored-By: Claude Opus 5 --- .../com/wytehat/btlogger/MapActivity.java | 229 +++++++++- .../com/wytehat/btlogger/ServerUploader.java | 428 ++++++++++++++++++ 2 files changed, 648 insertions(+), 9 deletions(-) create mode 100644 app/src/main/java/com/wytehat/btlogger/ServerUploader.java diff --git a/app/src/main/java/com/wytehat/btlogger/MapActivity.java b/app/src/main/java/com/wytehat/btlogger/MapActivity.java index 368809b..4547488 100644 --- a/app/src/main/java/com/wytehat/btlogger/MapActivity.java +++ b/app/src/main/java/com/wytehat/btlogger/MapActivity.java @@ -54,8 +54,11 @@ private TextView normalizationStatsText; private Button applyButton; private Button resetButton; -private Button exportRawButton; -private Button exportNormalizedButton; +private Button exportRawButton; +private Button exportNormalizedButton; +private Button uploadButton; +private Button serverSettingsButton; +private TextView uploadStatusText; private final ArrayList deviceValues = new ArrayList(); @@ -842,7 +845,63 @@ private void buildFilterPanel() { weightParams() ); - filterPanel.addView(exportRow); + filterPanel.addView(exportRow); + + LinearLayout serverRow = + new LinearLayout(this); + + serverRow.setOrientation( + LinearLayout.HORIZONTAL + ); + + uploadButton = + new Button(this); + + uploadButton.setText( + "Upload to Server" + ); + + serverSettingsButton = + new Button(this); + + serverSettingsButton.setText( + "Server…" + ); + + serverRow.addView( + uploadButton, + new LinearLayout.LayoutParams( + 0, + dp(40), + 2 + ) + ); + + serverRow.addView( + serverSettingsButton, + weightParams() + ); + + filterPanel.addView(serverRow); + + uploadStatusText = + new TextView(this); + + uploadStatusText.setTextSize(11); + uploadStatusText.setTextColor(Color.GRAY); + + uploadStatusText.setPadding( + dp(4), + dp(2), + dp(4), + dp(4) + ); + + uploadStatusText.setText( + lastUploadSummary() + ); + + filterPanel.addView(uploadStatusText); applyButton.setOnClickListener( new View.OnClickListener() { @@ -929,7 +988,25 @@ private void buildFilterPanel() { } ); - exportNormalizedButton.setOnClickListener( + uploadButton.setOnClickListener( + new View.OnClickListener() { + @Override + public void onClick(View v) { + uploadToServer(); + } + } + ); + + serverSettingsButton.setOnClickListener( + new View.OnClickListener() { + @Override + public void onClick(View v) { + showServerSettings(); + } + } + ); + + exportNormalizedButton.setOnClickListener( new View.OnClickListener() { @Override public void onClick(View v) { @@ -1787,11 +1864,145 @@ private boolean isInsideFilter( filterEnd.getTimeInMillis(); } -/* - * ================================================================ - * EXPORT - * ================================================================ - */ +/* + * ================================================================ + * SERVER UPLOAD + * ================================================================ + */ + +private String lastUploadSummary() { + + long last = + ServerUploader.lastUploadAt(this); + + if (last <= 0) + return "Never uploaded to " + + ServerUploader.serverUrl(this); + + return "Last upload: " + + MapFormat.date(last); +} + +/* + * Sends every recorded sighting. The server deduplicates on + * (mac, timestamp, lat, lon), so re-sending the whole database is + * idempotent -- anything it already holds comes back counted as a + * duplicate rather than stored twice. That is what makes a failed + * or partial upload safe to simply retry. + */ +private void uploadToServer() { + + uploadButton.setEnabled(false); + + uploadStatusText.setText( + "Uploading to " + + ServerUploader.serverUrl(this) + + "…" + ); + + ServerUploader.uploadAsync( + this, + new ServerUploader.Callback() { + @Override + public void onFinished( + ServerUploader.Result result) { + + uploadButton.setEnabled(true); + + uploadStatusText.setText( + result.summary() + ); + + android.widget.Toast.makeText( + MapActivity.this, + result.summary(), + android.widget.Toast.LENGTH_LONG + ).show(); + } + } + ); +} + +private void showServerSettings() { + + LinearLayout layout = + new LinearLayout(this); + + layout.setOrientation( + LinearLayout.VERTICAL + ); + + layout.setPadding( + dp(16), + dp(8), + dp(16), + dp(8) + ); + + layout.addView( + filterLabel("Server upload URL") + ); + + final EditText urlField = + new EditText(this); + + urlField.setText( + ServerUploader.serverUrl(this) + ); + + urlField.setTextSize(13); + urlField.setSingleLine(true); + + layout.addView(urlField); + + layout.addView( + filterLabel("API key") + ); + + final EditText keyField = + new EditText(this); + + keyField.setText( + ServerUploader.apiKey(this) + ); + + keyField.setTextSize(13); + keyField.setSingleLine(true); + + layout.addView(keyField); + + new AlertDialog.Builder(this) + .setTitle("Server settings") + .setView(layout) + .setPositiveButton( + "Save", + new DialogInterface.OnClickListener() { + @Override + public void onClick( + DialogInterface dialog, + int which) { + + ServerUploader.saveSettings( + MapActivity.this, + urlField.getText().toString(), + keyField.getText().toString() + ); + + uploadStatusText.setText( + lastUploadSummary() + ); + } + } + ) + .setNegativeButton("Cancel", null) + .show(); +} + +/* + * ================================================================ + * EXPORT + * ================================================================ + */ private void exportFilteredData( boolean normalized) { diff --git a/app/src/main/java/com/wytehat/btlogger/ServerUploader.java b/app/src/main/java/com/wytehat/btlogger/ServerUploader.java new file mode 100644 index 0000000..5f28750 --- /dev/null +++ b/app/src/main/java/com/wytehat/btlogger/ServerUploader.java @@ -0,0 +1,428 @@ +package com.wytehat.btlogger; + +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Handler; +import android.os.Looper; + +import org.json.JSONObject; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; + +/** + * Ships recorded sightings to the ingestion server as a CSV upload. + * + * The server deduplicates on (mac, timestamp, latitude, longitude), so + * re-uploading the whole database is safe and idempotent - anything it has + * already seen is counted and skipped. That is why this uploads everything + * rather than trying to track what was sent last time: the client staying + * dumb is what makes a failed or partial upload harmless to retry. + * + * Network work must not run on the main thread; use {@link #uploadAsync}, or + * call {@link #upload} from a background thread yourself. + */ +public final class ServerUploader { + + public static final String PREFS = "server_settings"; + + public static final String KEY_URL = "server_url"; + public static final String KEY_API_KEY = "server_api_key"; + public static final String KEY_LAST_UPLOAD = "last_upload_at"; + + public static final String DEFAULT_URL = + "https://bt.justbug.me/api/v1/bt/upload"; + + public static final String DEFAULT_API_KEY = + "android-app-key-12345-change-in-production"; + + /** Columns the server's importer recognises without any aliasing. */ + private static final String HEADER = + "mac_address,device_name,device_class,rssi,latitude,longitude,timestamp"; + + private static final int CONNECT_TIMEOUT_MS = 20000; + private static final int READ_TIMEOUT_MS = 60000; + + /** Outcome of one upload, mirroring the server's response metrics. */ + public static class Result { + + public boolean ok; + public int httpStatus; + + public int totalReceived; + public int newPointsAdded; + public int duplicatesSkipped; + public int invalidRows; + + public int rowsSent; + public String message = ""; + + /** One line suitable for a toast or a status label. */ + public String summary() { + + if (!ok) return "Upload failed: " + message; + + return "Uploaded " + rowsSent + " row(s) — " + + newPointsAdded + " new, " + + duplicatesSkipped + " already known" + + (invalidRows > 0 ? ", " + invalidRows + " rejected" : "") + + "."; + } + } + + public interface Callback { + void onFinished(Result result); + } + + private ServerUploader() { + } + + // ------------------------------------------------------------------ + // Settings + // ------------------------------------------------------------------ + + public static String serverUrl(Context context) { + return prefs(context).getString(KEY_URL, DEFAULT_URL); + } + + public static String apiKey(Context context) { + return prefs(context).getString(KEY_API_KEY, DEFAULT_API_KEY); + } + + public static long lastUploadAt(Context context) { + return prefs(context).getLong(KEY_LAST_UPLOAD, 0L); + } + + public static void saveSettings(Context context, String url, String key) { + + prefs(context).edit() + .putString(KEY_URL, url == null || url.trim().length() == 0 + ? DEFAULT_URL : url.trim()) + .putString(KEY_API_KEY, key == null || key.trim().length() == 0 + ? DEFAULT_API_KEY : key.trim()) + .apply(); + } + + private static SharedPreferences prefs(Context context) { + return context.getSharedPreferences(PREFS, 0); + } + + // ------------------------------------------------------------------ + // CSV + // ------------------------------------------------------------------ + + /** + * Every stored sighting as CSV: one row per device's current fix, plus one + * row per recorded history point, which is what carries the trail. + */ + public static String buildCsv(TrackerDatabase database) { + + StringBuilder csv = new StringBuilder(HEADER).append('\n'); + + SimpleDateFormat stamp = + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); + + // The server reads these as UTC; sending local time would shift every + // point by the phone's offset. + stamp.setTimeZone(TimeZone.getTimeZone("UTC")); + + List devices = database.all(); + + for (int i = 0; i < devices.size(); i++) { + + DeviceRecord record = devices.get(i); + + if (record == null || record.address == null) continue; + + String name = record.displayName(); + String category = record.category; + + if (record.hasLocation) { + + appendRow(csv, stamp, record.address, name, category, + record.rssi != 0 ? Integer.toString(record.rssi) : "", + record.latitude, record.longitude, + record.updatedAt > 0 + ? record.updatedAt + : System.currentTimeMillis()); + } + + List history = database.history(record.address, 0L); + + for (int p = 0; p < history.size(); p++) { + + LocationPoint point = history.get(p); + + if (point == null) continue; + + long time = point.timestamp > 0 + ? point.timestamp + : record.updatedAt; + + if (time <= 0) continue; + + // History rows carry no RSSI of their own - the column stays + // empty rather than repeating the device's latest reading, + // which was measured somewhere else entirely. + appendRow(csv, stamp, record.address, name, category, "", + point.latitude, point.longitude, time); + } + } + + return csv.toString(); + } + + private static void appendRow(StringBuilder csv, SimpleDateFormat stamp, + String mac, String name, String category, + String rssi, double latitude, + double longitude, long time) { + + csv.append(field(mac)).append(',') + .append(field(name)).append(',') + .append(field(category)).append(',') + .append(rssi).append(',') + .append(latitude).append(',') + .append(longitude).append(',') + .append(stamp.format(new Date(time))) + .append('\n'); + } + + /** Quotes a CSV field only when it needs it. */ + private static String field(String value) { + + if (value == null) return ""; + + if (value.indexOf(',') < 0 + && value.indexOf('"') < 0 + && value.indexOf('\n') < 0 + && value.indexOf('\r') < 0) { + + return value; + } + + return '"' + value.replace("\"", "\"\"") + .replace("\r", " ") + .replace("\n", " ") + '"'; + } + + /** Rows in a CSV body, excluding the header. */ + private static int countRows(String csv) { + + int rows = 0; + + for (int i = 0; i < csv.length(); i++) { + if (csv.charAt(i) == '\n') rows++; + } + + return Math.max(0, rows - 1); + } + + // ------------------------------------------------------------------ + // Upload + // ------------------------------------------------------------------ + + /** Builds the CSV and POSTs it. Blocking: never call this on the UI thread. */ + public static Result upload(Context context) { + + Result result = new Result(); + + TrackerDatabase database = null; + + String csv; + + try { + + database = new TrackerDatabase(context); + csv = buildCsv(database); + + } catch (Exception error) { + + result.message = "Could not read the local database: " + + error.getMessage(); + return result; + + } finally { + + if (database != null) { + try { database.close(); } catch (Exception ignored) { } + } + } + + result.rowsSent = countRows(csv); + + if (result.rowsSent == 0) { + result.message = "Nothing recorded yet — nothing to upload."; + return result; + } + + return post(context, csv, result); + } + + private static Result post(Context context, String csv, Result result) { + + String boundary = "----BTLogger" + System.currentTimeMillis(); + + HttpURLConnection connection = null; + + try { + + URL endpoint = new URL(serverUrl(context)); + + connection = (HttpURLConnection) endpoint.openConnection(); + + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setConnectTimeout(CONNECT_TIMEOUT_MS); + connection.setReadTimeout(READ_TIMEOUT_MS); + + connection.setRequestProperty("X-API-Key", apiKey(context)); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("Content-Type", + "multipart/form-data; boundary=" + boundary); + + // Streamed rather than buffered so a large export does not have to + // fit in memory twice on a phone. + connection.setChunkedStreamingMode(16 * 1024); + + DataOutputStream body = + new DataOutputStream(connection.getOutputStream()); + + body.writeBytes("--" + boundary + "\r\n"); + body.writeBytes("Content-Disposition: form-data; name=\"file\"; " + + "filename=\"btlogger.csv\"\r\n"); + body.writeBytes("Content-Type: text/csv\r\n\r\n"); + + body.write(csv.getBytes("UTF-8")); + + body.writeBytes("\r\n--" + boundary + "--\r\n"); + + body.flush(); + body.close(); + + result.httpStatus = connection.getResponseCode(); + + String response = readAll(result.httpStatus >= 400 + ? connection.getErrorStream() + : connection.getInputStream()); + + if (result.httpStatus == 401) { + result.message = "Server rejected the API key."; + return result; + } + + if (result.httpStatus >= 400) { + result.message = "HTTP " + result.httpStatus + + (response.length() > 0 ? ": " + trim(response) : ""); + return result; + } + + parseMetrics(response, result); + + result.ok = true; + + prefs(context).edit() + .putLong(KEY_LAST_UPLOAD, System.currentTimeMillis()) + .apply(); + + return result; + + } catch (Exception error) { + + result.message = error.getClass().getSimpleName() + + (error.getMessage() != null ? ": " + error.getMessage() : ""); + + return result; + + } finally { + + if (connection != null) connection.disconnect(); + } + } + + private static void parseMetrics(String response, Result result) { + + try { + + JSONObject json = new JSONObject(response); + + result.totalReceived = json.optInt("total_received", 0); + result.newPointsAdded = json.optInt("new_points_added", 0); + result.duplicatesSkipped = json.optInt("duplicates_skipped", 0); + result.invalidRows = json.optInt("invalid_rows", 0); + + } catch (Exception ignored) { + // A 2xx with an unreadable body still means the server took it. + } + } + + private static String readAll(InputStream stream) { + + if (stream == null) return ""; + + StringBuilder text = new StringBuilder(); + + BufferedReader reader = null; + + try { + + reader = new BufferedReader(new InputStreamReader(stream, "UTF-8")); + + String line; + + while ((line = reader.readLine()) != null) { + text.append(line); + } + + } catch (Exception ignored) { + + } finally { + + if (reader != null) { + try { reader.close(); } catch (Exception ignored) { } + } + } + + return text.toString(); + } + + private static String trim(String value) { + return value.length() > 200 ? value.substring(0, 200) + "…" : value; + } + + /** + * Runs {@link #upload} on a background thread and delivers the result on + * the main thread. + */ + public static void uploadAsync(final Context context, + final Callback callback) { + + final Context application = context.getApplicationContext(); + final Handler main = new Handler(Looper.getMainLooper()); + + new Thread(new Runnable() { + @Override + public void run() { + + final Result result = upload(application); + + if (callback == null) return; + + main.post(new Runnable() { + @Override + public void run() { + callback.onFinished(result); + } + }); + } + }, "bt-upload").start(); + } +}