""" auto_label.py — Auto-classify crops + Web Review UI ===================================================== Runs the full pipeline in one command: Step 1 Run YOLOv8n on your source images → crop every detection Step 2 Run the classifier on every crop → propose a class label Step 3 Open a local web UI so you review, confirm, fix, or reject Step 4 On "Commit", write the final data/ folder ready for 1b_split_dataset.py Output layout (after commit) ----------------------------- data/ cola_can/ ← confirmed crops pepsi_can/ … _rejected_/ ← crops you marked as reject _unreviewed_/ ← anything you closed without reviewing Usage ----- python auto_label.py \ --source shelf_images/ \ --detector_weights detector/best.pt \ --classifier_weights runs/classify/best.pt \ --output_dir data \ --det_conf 0.25 \ --cls_conf 0.50 \ --port 5000 """ import argparse import base64 import json import os import shutil import sys import threading import time import webbrowser from pathlib import Path import cv2 import numpy as np import torch import torch.nn.functional as F from flask import Flask, jsonify, request, send_file from torchvision import models, transforms from tqdm import tqdm from ultralytics import YOLO # ─────────────────────────── args ──────────────────────────────────────────── def parse_args(): p = argparse.ArgumentParser(description="Auto-label crops and review in browser") p.add_argument("--source", required=True) p.add_argument("--detector_weights", default="detector/best.pt") p.add_argument("--classifier_weights", default="runs/classify/best.pt") p.add_argument("--output_dir", default="data") p.add_argument("--staging_dir", default=".autolabel_staging", help="Temp folder for staged crops before commit (hidden by default)") p.add_argument("--det_conf", type=float, default=0.25) p.add_argument("--det_iou", type=float, default=0.45) p.add_argument("--cls_conf", type=float, default=0.75) p.add_argument("--img_size", type=int, default=640) p.add_argument("--padding", type=int, default=10) p.add_argument("--min_size", type=int, default=32) p.add_argument("--port", type=int, default=5000) return p.parse_args() ARGS = parse_args() # ─────────────────────────── device / OOM ──────────────────────────────────── def get_device(): if torch.cuda.is_available(): return torch.device("cuda") if torch.backends.mps.is_available(): return torch.device("mps") return torch.device("cpu") DEVICE = get_device() def _is_oom(exc): if isinstance(exc, MemoryError): return True if torch.cuda.is_available() and isinstance(exc, torch.cuda.OutOfMemoryError): return True if isinstance(exc, RuntimeError): m = str(exc).lower() return "out of memory" in m or ("memory" in m and "alloc" in m) return False def _free_cache(): if DEVICE.type == "cuda": torch.cuda.empty_cache() elif DEVICE.type == "mps" and hasattr(torch.mps, "empty_cache"): torch.mps.empty_cache() # ─────────────────────────── classifier ────────────────────────────────────── class ProductClassifier: def __init__(self, weights_path, device, img_size=224): ckpt = torch.load(weights_path, map_location=device) self.class_names = ckpt["class_names"] n = len(self.class_names) name = ckpt.get("args", {}).get("model", "efficientnet_b0") m = { "efficientnet_b0": models.efficientnet_b0, "efficientnet_b2": models.efficientnet_b2, "mobilenet_v3_small": models.mobilenet_v3_small, "resnet50": models.resnet50, }[name](weights=None) if "efficientnet" in name: m.classifier[1] = torch.nn.Linear(m.classifier[1].in_features, n) elif "mobilenet" in name: m.classifier[3] = torch.nn.Linear(m.classifier[3].in_features, n) else: m.fc = torch.nn.Linear(m.fc.in_features, n) m.load_state_dict(ckpt["model_state"]) self.model = m.to(device).eval() self.device = device self.tf = transforms.Compose([ transforms.ToPILImage(), transforms.Resize(int(img_size * 1.15)), transforms.CenterCrop(img_size), transforms.ToTensor(), transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225]), ]) self._bs = self._probe(img_size) def _probe(self, sz): if self.device.type == "cpu": return 64 dummy = torch.zeros(1,3,sz,sz,device=self.device) safe, probe = 1, 2 with torch.no_grad(): while probe <= 512: try: self.model(dummy.expand(probe,-1,-1,-1)) safe, probe = probe, probe*2 except Exception as e: if _is_oom(e): break raise finally: _free_cache() return max(1, int(safe*0.8)) @torch.no_grad() def predict(self, crops_bgr): if not crops_bgr: return [] tensors = [self.tf(cv2.cvtColor(c, cv2.COLOR_BGR2RGB)) for c in crops_bgr] results, i, bs = [], 0, self._bs while i < len(tensors): batch = torch.stack(tensors[i:i+bs]).to(self.device) try: probs = F.softmax(self.model(batch), dim=1) confs, idxs = probs.max(dim=1) # Return top-3 predictions per crop for the review UI top3_probs, top3_idxs = torch.topk(probs, min(3, probs.shape[1]), dim=1) for j in range(len(idxs)): results.append({ "class": self.class_names[idxs[j].item()], "confidence": round(confs[j].item(), 4), "top3": [ {"class": self.class_names[top3_idxs[j][k].item()], "conf": round(top3_probs[j][k].item(), 4)} for k in range(top3_probs.shape[1]) ] }) i += bs except Exception as e: if not _is_oom(e): raise _free_cache() bs = self._bs = max(1, bs // 2) finally: del batch return results # ─────────────────────────── pipeline ──────────────────────────────────────── IMAGE_EXTS = {".jpg",".jpeg",".png",".bmp",".webp",".tiff"} def get_images(source): p = Path(source) if p.is_file() and p.suffix.lower() in IMAGE_EXTS: return [p] if p.is_dir(): imgs = sorted(f for f in p.rglob("*") if f.suffix.lower() in IMAGE_EXTS) if not imgs: raise FileNotFoundError(f"No images in {p}") return imgs raise FileNotFoundError(f"Cannot open source: {source}") def img_to_b64(arr): _, buf = cv2.imencode(".jpg", arr, [cv2.IMWRITE_JPEG_QUALITY, 88]) return base64.b64encode(buf).decode() def run_pipeline(detector, classifier, args): """ Detect → crop → classify → stage. Returns list of crop metadata dicts that the review UI reads. """ staging = Path(args.staging_dir) if staging.exists(): shutil.rmtree(staging) staging.mkdir(parents=True) image_paths = get_images(args.source) print(f"\n[pipeline] {len(image_paths)} source image(s) found") crops_meta = [] # master list for the review UI total_det = total_skip = 0 for img_path in tqdm(image_paths, desc="Detecting"): frame = cv2.imread(str(img_path)) if frame is None: print(f" ⚠ Cannot read {img_path.name}") continue h, w = frame.shape[:2] results = detector(frame, conf=args.det_conf, iou=args.det_iou, imgsz=args.img_size, verbose=False)[0] boxes, crops_bgr = [], [] for box in results.boxes: x1,y1,x2,y2 = map(int, box.xyxy[0].tolist()) x1=max(0,x1-args.padding); y1=max(0,y1-args.padding) x2=min(w,x2+args.padding); y2=min(h,y2+args.padding) crop = frame[y1:y2, x1:x2] if crop.size==0 or (x2-x1)/ cls_dir = staging / cls_name cls_dir.mkdir(exist_ok=True) crop_path = cls_dir / f"{crop_id}.jpg" cv2.imwrite(str(crop_path), crop) crops_meta.append({ "id": crop_id, "file": str(crop_path), "source_image": img_path.name, "pred_class": cls_name, "confidence": conf, "top3": pred["top3"], "bbox": list(box), # review fields (mutated by the UI) "label": cls_name, # user's chosen label "status": "auto", # "auto" | "confirmed" | "rejected" | "reclassified" }) total_det += 1 print(f"[pipeline] {total_det} crops staged | {total_skip} skipped") return crops_meta # ─────────────────────────── Flask app ─────────────────────────────────────── flask_app = Flask(__name__) STATE = { "crops": [], # list of crop metadata dicts "classes": [], # classifier class names "done": False, # set to True when user commits } @flask_app.route("/") def index(): return HTML_UI @flask_app.route("/api/crops") def api_crops(): return jsonify(STATE["crops"]) @flask_app.route("/api/classes") def api_classes(): return jsonify(STATE["classes"]) @flask_app.route("/api/crop_image/") def crop_image(crop_id): crop = next((c for c in STATE["crops"] if c["id"] == crop_id), None) if not crop: return "not found", 404 return send_file(crop["file"], mimetype="image/jpeg") @flask_app.route("/api/update", methods=["POST"]) def api_update(): """Receive bulk label updates from the UI.""" updates = request.json # list of {id, label, status} lookup = {c["id"]: c for c in STATE["crops"]} for u in updates: if u["id"] in lookup: lookup[u["id"]]["label"] = u["label"] lookup[u["id"]]["status"] = u["status"] return jsonify({"ok": True}) @flask_app.route("/api/commit", methods=["POST"]) def api_commit(): """Move staged crops to final output_dir based on user decisions.""" out_root = Path(ARGS.output_dir) moved = rejected = unreviewed = 0 for crop in STATE["crops"]: src = Path(crop["file"]) if not src.exists(): continue status = crop["status"] label = crop["label"] if status == "rejected": dst_dir = out_root / "_rejected_" elif status == "unreviewed" or status == "auto": dst_dir = out_root / "_unreviewed_" else: dst_dir = out_root / label dst_dir.mkdir(parents=True, exist_ok=True) shutil.copy2(str(src), str(dst_dir / src.name)) if status == "rejected": rejected += 1 elif status in ("auto","unreviewed"): unreviewed += 1 else: moved += 1 summary = { "committed": moved, "rejected": rejected, "unreviewed": unreviewed, "output_dir": str(out_root.resolve()), } STATE["done"] = True return jsonify(summary) @flask_app.route("/api/stats") def api_stats(): crops = STATE["crops"] by_class = {} for c in crops: by_class.setdefault(c["pred_class"], 0) by_class[c["pred_class"]] += 1 statuses = {} for c in crops: statuses[c["status"]] = statuses.get(c["status"], 0) + 1 return jsonify({"by_class": by_class, "by_status": statuses, "total": len(crops)}) # ─────────────────────────── embedded UI ───────────────────────────────────── HTML_UI = r""" Auto-Label Review

Auto-Label Review

Loading crops…

⬡ AutoLabel

Status:
Min conf: 0.00
0 selected
""" # ─────────────────────────── entry point ───────────────────────────────────── def main(): args = ARGS print(f"[startup] Device : {DEVICE}") print(f"[startup] Detector : {args.detector_weights}") print(f"[startup] Classifier : {args.classifier_weights}") detector = YOLO(args.detector_weights) classifier = ProductClassifier(args.classifier_weights, DEVICE) print("\n[step 1/2] Running detection + auto-classification …") crops = run_pipeline(detector, classifier, args) if not crops: print("⚠ No crops were generated. Check --source and --det_conf.") sys.exit(1) STATE["crops"] = crops STATE["classes"] = classifier.class_names print(f"\n[step 2/2] Launching review UI on http://localhost:{args.port}") print(" Keyboard shortcuts inside a crop:") print(" C → confirm") print(" R → reject") print(" ← → → previous / next crop") print(" Esc → close modal") print(" Ctrl/Shift+click → multi-select for bulk actions") print("\n Close the browser tab or press Ctrl+C to stop.\n") # Open browser after a short delay so Flask is ready def _open(): time.sleep(1.2) webbrowser.open(f"http://localhost:{args.port}") threading.Thread(target=_open, daemon=True).start() # Run Flask (single-threaded to keep GPU access safe) flask_app.run(host="0.0.0.0", port=args.port, threaded=False, use_reloader=False) if __name__ == "__main__": main()