From 0f8cf2ff5ca722186d2e9de8f2395f29681b5428 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Fri, 14 Mar 2025 21:02:08 +0000 Subject: [PATCH] research: Add roi.py --- research/fetch | 2 + research/qrdet.py | 0 .../research-tools/src/views/FolderView.vue | 71 +++++++++++++- research/roi.py | 98 +++++++++++++++++++ research/server.py | 11 ++- 5 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 research/qrdet.py create mode 100755 research/roi.py diff --git a/research/fetch b/research/fetch index d60c762..4b16faa 100755 --- a/research/fetch +++ b/research/fetch @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import os import argparse import subprocess @@ -16,6 +17,7 @@ def main(): # we cannot rsync to data/json because we will update there after sync, and # the next sync will overwrite the updated files local_dir = f'data/raw/camera-frame' + os.makedirs(local_dir, exist_ok=True) cmd = ['rsync', '-avz', f'zy:{remote_dir}', local_dir] subprocess.run(cmd) diff --git a/research/qrdet.py b/research/qrdet.py new file mode 100644 index 0000000..e69de29 diff --git a/research/research-tools/src/views/FolderView.vue b/research/research-tools/src/views/FolderView.vue index 1ccaadd..1a17e1e 100644 --- a/research/research-tools/src/views/FolderView.vue +++ b/research/research-tools/src/views/FolderView.vue @@ -25,6 +25,21 @@ max="1000" step="10"> +
+ Total: {{ total }} + + Page {{ currentPage }} of {{ totalPages }} + + +
@@ -68,11 +83,26 @@ export default { selectionMode: false, isAnalyzing: false, showDropdown: false, + total: 0, + start: 0, + pageSize: 20, + } + }, + computed: { + currentPage() { + return Math.floor(this.start / this.pageSize) + 1; + }, + totalPages() { + return Math.ceil(this.total / this.pageSize); } }, watch: { imageSize(newSize) { localStorage.setItem('imageSize', newSize); + }, + pageSize() { + this.start = 0; // Reset to first page when changing page size + this.reload(); } }, mounted() { @@ -80,11 +110,14 @@ export default { }, methods: { async reload() { - const res = await fetch(`/api/datapoints?folder=${this.folder}`); + this.datapoints = []; + const res = await fetch(`/api/datapoints?folder=${this.folder}&start=${this.start}&count=${this.pageSize}`); const data = await res.json(); - console.log(data); this.datapoints = data.datapoints.sort((a, b) => a.session_id - b.session_id); + this.total = data.total; + // Initialize selection state for new datapoints + this.selectedDatapoints = {}; for (const datapoint of this.datapoints) { this.selectedDatapoints[datapoint.path] = false; } @@ -196,6 +229,14 @@ export default { this.isAnalyzing = false; } }, + nextPage() { + this.start += this.pageSize; + this.reload(); + }, + previousPage() { + this.start = Math.max(0, this.start - this.pageSize); + this.reload(); + }, }, } @@ -345,4 +386,30 @@ div.datapoints img { .qr-info > div { margin: 2px 0; } + +.pagination-controls { + display: flex; + align-items: center; + gap: 1rem; + margin-top: 1rem; +} + +.pagination-controls button { + padding: 0.5rem 1rem; + background-color: #4CAF50; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; +} + +.pagination-controls button:disabled { + background-color: #cccccc; + cursor: not-allowed; +} + +.pagination-controls select { + padding: 0.5rem; + border-radius: 4px; +} \ No newline at end of file diff --git a/research/roi.py b/research/roi.py new file mode 100755 index 0000000..580d292 --- /dev/null +++ b/research/roi.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +import argparse +import os +import requests +import json +import subprocess + +class RoiResearch(object): + def __init__(self, token=None): + self.token = token + + def login(self, username, password): + url = "https://themblem.com/api/v1/login/" + data = { + "username": username, + "password": password, + } + response = requests.post(url, json=data) + token = response.json()['token'] + self.token = token + return token + + def fetch_scan_data(self, last_id=None): + x = "/api/v1/scan-data/?limit=1000" + while True: + url = "https://themblem.com" + x + print("fetching", url) + response = requests.get(url, headers={"Authorization": f"Token {self.token}"}) + r = response.json() + meta = r['meta'] + if meta['next'] is None: + break + x = meta['next'] + for sd in r['objects']: + if last_id and sd['id'] <= last_id: + return + yield sd + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--token", '-t', default='3ebd8c33-f46e-4b06-bda8-4c0f5f5eb530', type=str) + parser.add_argument("--username", "-u", type=str) + parser.add_argument("--password", "-p", type=str) + return parser.parse_args() + +def get_last_id(list_file): + ret = None + if not os.path.exists(list_file): + return ret + with open(list_file, "r") as f: + for line in f: + if not line.strip(): + continue + id = json.loads(line)['id'] + if not ret or id > ret: + ret = id + return ret + +def get_all_ids(list_file): + ret = [] + with open(list_file, "r") as f: + for line in f: + ret.append(json.loads(line)['id']) + return ret + +def main(): + args = parse_args() + rr = RoiResearch(args.token) + if args.username and args.password: + print(rr.login(args.username, args.password)) + list_file = "data/roi/list.txt" + last_id = get_last_id(list_file) + print("last_id", last_id) + with open(list_file, "a") as f: + for sd in rr.fetch_scan_data(last_id): + if not sd['image'] or not sd['labels']: + print("skipping", sd['id']) + continue + print("new id", sd['id']) + line = json.dumps({ + 'id': sd['id'], + 'image': sd['image'], + 'labels': sd['labels'], + }) + f.write(line + "\n") + all_ids = get_all_ids(list_file) + for id in all_ids: + outd = f"data/roi/samples/{id}" + if os.path.exists(f"{outd}/{id}.json"): + print(f"skipping {id}, json already exists") + continue + cmd = f'../scripts/emcli get-scan-data {id} -o {outd}' + print(cmd) + subprocess.call(cmd, shell=True) + +if __name__ == "__main__": + main() diff --git a/research/server.py b/research/server.py index 5ebef31..29e5d16 100755 --- a/research/server.py +++ b/research/server.py @@ -14,7 +14,7 @@ import numpy as np app = Flask(__name__) -DATA_DIR = os.environ.get('DATA_DIR', '/var/tmp/themblem-research') +DATA_DIR = os.environ.get('DATA_DIR', 'data') os.makedirs(DATA_DIR, exist_ok=True) def image_data_url_to_binary(image_data_url): @@ -101,8 +101,12 @@ def qrtool(): @app.route('/api/datapoints') def datapoints(): folder = request.args.get('folder') + start = int(request.args.get('start', 0)) + count = int(request.args.get('count', 10)) folder_dir = os.path.join(DATA_DIR, "json", folder) x = os.listdir(folder_dir) + total = len(x) + x = x[start:start+count] datapoints = [] for f in x: try: @@ -112,6 +116,9 @@ def datapoints(): return { "ok": True, "datapoints": datapoints, + "total": total, + "start": start, + "count": count, } def load_datapoint(path): @@ -230,4 +237,4 @@ def analyze(): return data if __name__ == '__main__': - app.run(host='0.0.0.0', port=26966, debug=True) \ No newline at end of file + app.run(host='0.0.0.0', port=26966, debug=True)