research: Add roi.py

research/fetch

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import argparse
 import subprocess
 
@@ -16,6 +17,7 @@ def main():
     # we cannot rsync to data/json because we will update there after sync, and
     # the next sync will overwrite the updated files
     local_dir = f'data/raw/camera-frame'
+    os.makedirs(local_dir, exist_ok=True)
     cmd = ['rsync', '-avz', f'zy:{remote_dir}', local_dir]
     subprocess.run(cmd)
 

research/research-tools/src/views/FolderView.vue

@@ -25,6 +25,21 @@
                     max="1000" 
                     step="10">
             </div>
+            <div class="pagination-controls">
+                <span>Total: {{ total }}</span>
+                <button 
+                    :disabled="start === 0" 
+                    @click="previousPage">Previous</button>
+                <span>Page {{ currentPage }} of {{ totalPages }}</span>
+                <button 
+                    :disabled="start + pageSize >= total" 
+                    @click="nextPage">Next</button>
+                <select v-model="pageSize">
+                    <option :value="20">20 per page</option>
+                    <option :value="50">50 per page</option>
+                    <option :value="100">100 per page</option>
+                </select>
+            </div>
         </div>
         <div class="datapoints">
             <div class="datapoint" v-for="dp in datapoints" :key="dp.path">
@@ -68,11 +83,26 @@ export default {
             selectionMode: false,
             isAnalyzing: false,
             showDropdown: false,
+            total: 0,
+            start: 0,
+            pageSize: 20,
+        }
+    },
+    computed: {
+        currentPage() {
+            return Math.floor(this.start / this.pageSize) + 1;
+        },
+        totalPages() {
+            return Math.ceil(this.total / this.pageSize);
         }
     },
     watch: {
         imageSize(newSize) {
             localStorage.setItem('imageSize', newSize);
+        },
+        pageSize() {
+            this.start = 0; // Reset to first page when changing page size
+            this.reload();
         }
     },
     mounted() {
@@ -80,11 +110,14 @@ export default {
     },
     methods: {
         async reload() {
-            const res = await fetch(`/api/datapoints?folder=${this.folder}`);
+            this.datapoints = [];
+            const res = await fetch(`/api/datapoints?folder=${this.folder}&start=${this.start}&count=${this.pageSize}`);
             const data = await res.json();
-            console.log(data);
             this.datapoints = data.datapoints.sort((a, b) => a.session_id - b.session_id);
+            this.total = data.total;
+            
             // Initialize selection state for new datapoints
+            this.selectedDatapoints = {};
             for (const datapoint of this.datapoints) {
                 this.selectedDatapoints[datapoint.path] = false;
             }
@@ -196,6 +229,14 @@ export default {
                 this.isAnalyzing = false;
             }
         },
+        nextPage() {
+            this.start += this.pageSize;
+            this.reload();
+        },
+        previousPage() {
+            this.start = Math.max(0, this.start - this.pageSize);
+            this.reload();
+        },
     },
 }
 </script>
@@ -345,4 +386,30 @@ div.datapoints img {
 .qr-info > div {
     margin: 2px 0;
 }
+
+.pagination-controls {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    margin-top: 1rem;
+}
+
+.pagination-controls button {
+    padding: 0.5rem 1rem;
+    background-color: #4CAF50;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+.pagination-controls button:disabled {
+    background-color: #cccccc;
+    cursor: not-allowed;
+}
+
+.pagination-controls select {
+    padding: 0.5rem;
+    border-radius: 4px;
+}
 </style>

research/roi.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import requests
+import json
+import subprocess
+
+class RoiResearch(object):
+    def __init__(self, token=None):
+        self.token = token
+
+    def login(self, username, password):
+        url = "https://themblem.com/api/v1/login/"
+        data = {
+            "username": username,
+            "password": password,
+        }
+        response = requests.post(url, json=data)
+        token = response.json()['token']
+        self.token = token
+        return token
+
+    def fetch_scan_data(self, last_id=None):
+        x = "/api/v1/scan-data/?limit=1000"
+        while True:
+            url = "https://themblem.com" + x
+            print("fetching", url)
+            response = requests.get(url, headers={"Authorization": f"Token {self.token}"})
+            r = response.json()
+            meta = r['meta']
+            if meta['next'] is None:
+                break
+            x = meta['next']
+            for sd in r['objects']:
+                if last_id and sd['id'] <= last_id:
+                    return
+                yield sd
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--token", '-t', default='3ebd8c33-f46e-4b06-bda8-4c0f5f5eb530', type=str)
+    parser.add_argument("--username", "-u", type=str)
+    parser.add_argument("--password", "-p", type=str)
+    return parser.parse_args()
+
+def get_last_id(list_file):
+    ret = None
+    if not os.path.exists(list_file):
+        return ret
+    with open(list_file, "r") as f:
+        for line in f:
+            if not line.strip():
+                continue
+            id = json.loads(line)['id']
+            if not ret or id > ret:
+                ret = id
+    return ret
+
+def get_all_ids(list_file):
+    ret = []
+    with open(list_file, "r") as f:
+        for line in f:
+            ret.append(json.loads(line)['id'])
+    return ret
+    
+def main():
+    args = parse_args()
+    rr = RoiResearch(args.token)
+    if args.username and args.password:
+        print(rr.login(args.username, args.password))
+    list_file = "data/roi/list.txt"
+    last_id = get_last_id(list_file)
+    print("last_id", last_id)
+    with open(list_file, "a") as f:
+        for sd in rr.fetch_scan_data(last_id):
+            if not sd['image'] or not sd['labels']:
+                print("skipping", sd['id'])
+                continue
+            print("new id", sd['id'])
+            line = json.dumps({
+                'id': sd['id'],
+                'image': sd['image'],
+                'labels': sd['labels'],
+            })
+            f.write(line + "\n")
+    all_ids = get_all_ids(list_file)
+    for id in all_ids:
+        outd = f"data/roi/samples/{id}"
+        if os.path.exists(f"{outd}/{id}.json"):
+            print(f"skipping {id}, json already exists")
+            continue
+        cmd = f'../scripts/emcli get-scan-data {id} -o {outd}'
+        print(cmd)
+        subprocess.call(cmd, shell=True)
+
+if __name__ == "__main__":
+    main()

research/server.py

@@ -14,7 +14,7 @@ import numpy as np
 
 app = Flask(__name__)
 
-DATA_DIR = os.environ.get('DATA_DIR', '/var/tmp/themblem-research')
+DATA_DIR = os.environ.get('DATA_DIR', 'data')
 os.makedirs(DATA_DIR, exist_ok=True)
 
 def image_data_url_to_binary(image_data_url):
@@ -101,8 +101,12 @@ def qrtool():
 @app.route('/api/datapoints')
 def datapoints():
     folder = request.args.get('folder')
+    start = int(request.args.get('start', 0))
+    count = int(request.args.get('count', 10))
     folder_dir = os.path.join(DATA_DIR, "json", folder)
     x = os.listdir(folder_dir)
+    total = len(x)
+    x = x[start:start+count]
     datapoints = []
     for f in x:
         try:
@@ -112,6 +116,9 @@ def datapoints():
     return {
         "ok": True,
         "datapoints": datapoints,
+        "total": total,
+        "start": start,
+        "count": count,
     }
 
 def load_datapoint(path):
@@ -230,4 +237,4 @@ def analyze():
     return data
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=26966, debug=True)
+    app.run(host='0.0.0.0', port=26966, debug=True)