run_prod.py

This commit is contained in:
Hamza committed 2026-06-03 11:07:23 +02:00
1 parent 83f80ab935
commit 85422a5bba
13 files changed
+50 -23

No files matched your search

+1 -1
View File
@@ -1 +1 @@
{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":16206274,"datasetId":10367968,"databundleVersionId":17185835}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"a1b2c3d4","cell_type":"markdown","source":"# 🛒 Product Classifier — Training Notebook\n\nTrains a lightweight **EfficientNet-B0** (ImageNet pre-trained) on shelf product crops\ngenerated by `1_generate_crop_dataset.py`.\n\n---\n\n### ✅ Before you run\n1. **Runtime → Change runtime type → GPU** (T4 is free and enough for this)\n2. Upload your `crops_dataset/` as a Kaggle Dataset and attach it via **+ Add Data**\n3. Update `DATA_DIR` in **Cell 2 — Configuration** to match your dataset path\n4. Run all cells top-to-bottom\n\n### 📂 Expected dataset layout\n```\n/kaggle/input/your-dataset-name/\n train/\n cola_can/ img001.jpg …\n pepsi_can/ img002.jpg …\n lays_chips/ img003.jpg …\n val/\n cola_can/ …\n …\n```\n\n### 📦 Outputs saved to `/kaggle/working/runs/classify/`\n| File | Description |\n|---|---|\n| `best.pt` | Best checkpoint (highest val accuracy) |\n| `last.pt` | Last epoch checkpoint |\n| `class_names.json` | `{idx: class_name}` mapping for inference |\n| `training_curves.png` | Loss & accuracy plots |","metadata":{}},{"id":"b2c3d4e5","cell_type":"code","source":"# ── Cell 1 — Install / verify dependencies ────────────────────────────────────\n# Kaggle kernels ship with torch & torchvision pre-installed.\n# We only need tqdm (already present) — nothing extra to install.\n\nimport sys, subprocess\n\ndef run(cmd):\n subprocess.run(cmd, shell=True, check=True)\n\n# Verify GPU is visible\nimport torch\nprint(f\"PyTorch : {torch.__version__}\")\nprint(f\"CUDA : {torch.version.cuda}\")\nprint(f\"GPU : {torch.cuda.get_device_name(0) if torch.cuda.is_available() else '⚠ NOT available — check runtime settings'}\")\nprint(f\"VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\" if torch.cuda.is_available() else \"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-11T09:57:04.799727Z","iopub.execute_input":"2026-05-11T09:57:04.800469Z","iopub.status.idle":"2026-05-11T09:57:04.805785Z","shell.execute_reply.started":"2026-05-11T09:57:04.800441Z","shell.execute_reply":"2026-05-11T09:57:04.805146Z"}},"outputs":[{"name":"stdout","text":"PyTorch : 2.10.0+cu128\nCUDA : 12.8\nGPU : Tesla T4\nVRAM : 15.6 GB\n","output_type":"stream"}],"execution_count":16},{"id":"c3d4e5f6","cell_type":"code","source":"# ── Cell 2 — Configuration (edit this cell) ──────────────────────────────────\nfrom pathlib import Path\n\n# ── Paths ──────────────────────────────────────────────────────────────────────\n# Point this to the root of your attached dataset (the folder that contains train/ and val/)\nDATA_DIR = Path(\"/kaggle/input/datasets/triztechdata/product-dataset/crops_dataset\") # ← CHANGE THIS\nOUTPUT_DIR = Path(\"/kaggle/working/runs/classify\")\n\n# ── Model ──────────────────────────────────────────────────────────────────────\n# Choices: \"efficientnet_b0\" | \"efficientnet_b2\" | \"mobilenet_v3_small\" | \"resnet50\"\nMODEL_NAME = \"efficientnet_b0\"\nFREEZE_BACKBONE = False # True = only train the head (faster, good for small datasets)\n\n# ── Training hyperparameters ───────────────────────────────────────────────────\nEPOCHS = 50\nBATCH_SIZE = 64\nIMG_SIZE = 224\nLR = 1e-3\nWEIGHT_DECAY = 1e-4\nPATIENCE = 10 # early-stop if val accuracy does not improve for this many epochs\nNUM_WORKERS = 2 # Kaggle kernels: keep ≤ 2 to avoid DataLoader warnings\nUSE_AMP = True # Automatic Mixed Precision — speeds up training on GPU\nSEED = 42\n\n# ── Sanity check ───────────────────────────────────────────────────────────────\nassert (DATA_DLine truncated
{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":16263031,"datasetId":10367968,"databundleVersionId":17247117}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"a1b2c3d4","cell_type":"markdown","source":"# 🛒 Product Classifier — Training Notebook\n\nTrains a lightweight **EfficientNet-B0** (ImageNet pre-trained) on shelf product crops\ngenerated by `1_generate_crop_dataset.py`.\n\n---\n\n### ✅ Before you run\n1. **Runtime → Change runtime type → GPU** (T4 is free and enough for this)\n2. Upload your `crops_dataset/` as a Kaggle Dataset and attach it via **+ Add Data**\n3. Update `DATA_DIR` in **Cell 2 — Configuration** to match your dataset path\n4. Run all cells top-to-bottom\n\n### 📂 Expected dataset layout\n```\n/kaggle/input/your-dataset-name/\n train/\n cola_can/ img001.jpg …\n pepsi_can/ img002.jpg …\n lays_chips/ img003.jpg …\n val/\n cola_can/ …\n …\n```\n\n### 📦 Outputs saved to `/kaggle/working/runs/classify/`\n| File | Description |\n|---|---|\n| `best.pt` | Best checkpoint (highest val accuracy) |\n| `last.pt` | Last epoch checkpoint |\n| `class_names.json` | `{idx: class_name}` mapping for inference |\n| `training_curves.png` | Loss & accuracy plots |","metadata":{}},{"id":"b2c3d4e5","cell_type":"code","source":"# ── Cell 1 — Install / verify dependencies ────────────────────────────────────\n# Kaggle kernels ship with torch & torchvision pre-installed.\n# We only need tqdm (already present) — nothing extra to install.\n\nimport sys, subprocess\n\ndef run(cmd):\n subprocess.run(cmd, shell=True, check=True)\n\n# Verify GPU is visible\nimport torch\nprint(f\"PyTorch : {torch.__version__}\")\nprint(f\"CUDA : {torch.version.cuda}\")\nprint(f\"GPU : {torch.cuda.get_device_name(0) if torch.cuda.is_available() else '⚠ NOT available — check runtime settings'}\")\nprint(f\"VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\" if torch.cuda.is_available() else \"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-14T12:15:41.201232Z","iopub.execute_input":"2026-05-14T12:15:41.201909Z","iopub.status.idle":"2026-05-14T12:15:46.397019Z","shell.execute_reply.started":"2026-05-14T12:15:41.201854Z","shell.execute_reply":"2026-05-14T12:15:46.396084Z"}},"outputs":[{"name":"stdout","text":"PyTorch : 2.10.0+cu128\nCUDA : 12.8\nGPU : Tesla T4\nVRAM : 15.6 GB\n","output_type":"stream"}],"execution_count":1},{"id":"c3d4e5f6","cell_type":"code","source":"# ── Cell 2 — Configuration (edit this cell) ──────────────────────────────────\nfrom pathlib import Path\n\n# ── Paths ──────────────────────────────────────────────────────────────────────\n# Point this to the root of your attached dataset (the folder that contains train/ and val/)\nDATA_DIR = Path(\"/kaggle/input/datasets/triztechdata/product-dataset/crops_dataset\") # ← CHANGE THIS\nOUTPUT_DIR = Path(\"/kaggle/working/runs/classify\")\n\n# ── Model ──────────────────────────────────────────────────────────────────────\n# Choices: \"efficientnet_b0\" | \"efficientnet_b2\" | \"mobilenet_v3_small\" | \"resnet50\"\nMODEL_NAME = \"efficientnet_b0\"\nFREEZE_BACKBONE = False # True = only train the head (faster, good for small datasets)\n\n# ── Training hyperparameters ───────────────────────────────────────────────────\nEPOCHS = 50\nBATCH_SIZE = 64\nIMG_SIZE = 224\nLR = 1e-3\nWEIGHT_DECAY = 1e-4\nPATIENCE = 10 # early-stop if val accuracy does not improve for this many epochs\nNUM_WORKERS = 2 # Kaggle kernels: keep ≤ 2 to avoid DataLoader warnings\nUSE_AMP = True # Automatic Mixed Precision — speeds up training on GPU\nSEED = 42\n\n# ── Sanity check ───────────────────────────────────────────────────────────────\nassert (DATA_DILine truncated
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 215 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 532 KiB

+37 -22
View File
@@ -33,7 +33,7 @@ Usage
# First time — creates the Kaggle dataset and notebook from scratch
python kaggle_train.py \
--dataset_dir crops_dataset \
--notebook train_classifier_kaggle.ipynb \
--notebook 2_product-classifier.ipynb \
--dataset_name my-product-crops \
--kernel_name product-classifier-train \
--output_dir runs/classify
@@ -41,16 +41,16 @@ python kaggle_train.py \
# Subsequent runs — detects existing dataset/kernel and updates them
python kaggle_train.py \
--dataset_dir crops_dataset \
--notebook train_classifier_kaggle.ipynb \
--notebook 2_product-classifier.ipynb \
--dataset_name my-product-crops \
--kernel_name product-classifier-train
Arguments
---------
--dataset_dir Local crops_dataset/ folder to upload (default: crops_dataset)
--notebook Kaggle notebook .ipynb file (default: train_classifier_kaggle.ipynb)
--dataset_name Kaggle dataset slug (lowercase, hyphens) (default: product-crops)
--kernel_name Kaggle kernel slug (lowercase, hyphens) (default: product-classifier)
--notebook Kaggle notebook .ipynb file (default: 2_product-classifier.ipynb)
--dataset_name Kaggle dataset slug (lowercase, hyphens) (default: product-dataset)
--kernel_name Kaggle kernel slug (lowercase, hyphens) (default: product-predection)
--output_dir Where to save downloaded weights (default: runs/classify)
--poll_interval Seconds between status checks (default: 30)
--timeout Max minutes to wait for kernel (default: 180)
@@ -79,10 +79,10 @@ def parse_args():
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p.add_argument("--dataset_dir", default="crops_dataset")
p.add_argument("--notebook", default="train_classifier_kaggle.ipynb")
p.add_argument("--dataset_name", default="product-crops",
p.add_argument("--notebook", default="2_product-classifier.ipynb")
p.add_argument("--dataset_name", default="product-dataset",
help="Kaggle dataset slug — lowercase, hyphens, no spaces")
p.add_argument("--kernel_name", default="product-classifier",
p.add_argument("--kernel_name", default="product-predection",
help="Kaggle kernel slug — lowercase, hyphens, no spaces")
p.add_argument("--output_dir", default="runs/classify")
p.add_argument("--poll_interval", type=int, default=30,
@@ -132,6 +132,7 @@ def _detect_auth_method() -> str:
if os.getenv("KAGGLE_USERNAME") and os.getenv("KAGGLE_KEY"):
return "env_legacy"
if _ACCESS_TOKEN_PATH.exists():
#print(f"this is the PATH: {_ACCESS_TOKEN_PATH}")
return "access_token"
if _KAGGLE_JSON_PATH.exists():
return "kaggle_json"
@@ -320,7 +321,7 @@ def patch_notebook(notebook_path: Path, dataset_ref: str,
DATA_DIR = Path("/kaggle/input/your-dataset-name")
We replace the path to match the actual uploaded dataset slug.
"""
nb = json.loads(notebook_path.read_text())
nb = json.loads(notebook_path.read_text(encoding="utf-8"))
# Extract the dataset slug from "username/slug" → "slug"
ds_slug = dataset_ref.split("/")[-1]
@@ -351,7 +352,7 @@ def patch_notebook(notebook_path: Path, dataset_ref: str,
log("⚠ DATA_DIR line not found in notebook — using notebook as-is", "⚠")
out_path = tmp_dir / notebook_path.name
out_path.write_text(json.dumps(nb, indent=1))
out_path.write_text(json.dumps(nb, indent=1), encoding="utf-8")
return out_path
@@ -389,11 +390,15 @@ def push_kernel(api, username: str, kernel_name: str, notebook_path: Path,
"competition_sources": [],
"kernel_sources": [],
"model_sources": [],
"machine_shape": "t4" if gpu else None,
}
(kernel_dir / "kernel-metadata.json").write_text(json.dumps(meta, indent=2))
log(f"Pushing kernel to Kaggle: {ref} (GPU={'yes' if gpu else 'no'})", "🚀")
api.kernels_push(str(kernel_dir))
try:
api.kernels_push(str(kernel_dir), timeout=None, acc="t4" if gpu else None)
except TypeError:
api.kernels_push(str(kernel_dir))
log(f"Kernel pushed — run started: https://www.kaggle.com/code/{ref}", "✓")
return ref
@@ -405,12 +410,11 @@ def poll_kernel(api, kernel_ref: str, poll_interval: int, timeout_minutes: int)
Poll the kernel until it reaches a terminal status.
Returns True if successful, False if failed or timed out.
"""
username, slug = kernel_ref.split("/")
deadline = time.time() + timeout_minutes * 60
attempt = 0
TERMINAL = {"complete", "error", "cancelled"}
RUNNING = {"running", "queued"}
TERMINAL = {"complete", "error", "cancelled", "cancel_acknowledged"}
RUNNING = {"running", "queued", "initializing", "pending"}
log(f"Polling kernel status every {poll_interval}s "
f"(timeout {timeout_minutes} min) …", "⏳")
@@ -418,16 +422,27 @@ def poll_kernel(api, kernel_ref: str, poll_interval: int, timeout_minutes: int)
while time.time() < deadline:
attempt += 1
try:
status_obj = api.kernel_status(username, slug)
# The API returns an object with a .status attribute
status = getattr(status_obj, "status", str(status_obj)).lower()
# kernels_status takes the full kernel ref "username/slug"
status_obj = api.kernels_status(kernel_ref)
# Extract status from the response object
raw_status = getattr(status_obj, "status", None)
if raw_status is not None:
if hasattr(raw_status, 'name'):
status = raw_status.name.lower()
else:
status = str(raw_status).lower()
else:
status = str(status_obj).lower()
# Clean up enum prefixes
status = status.replace("kernelworkerstatus.", "")
except Exception as e:
log(f"Poll error (will retry): {e}", "⚠")
time.sleep(poll_interval)
continue
elapsed_min = (time.time() + timeout_minutes * 60 - deadline) / 60 + timeout_minutes
print(f" [{attempt:>3}] status={status:<12} elapsed={elapsed_min:.1f} min",
elapsed_min = (time.time() + timeout_minutes * 60 - deadline) / 60
print(f" [{attempt:>3}] status={status:<20} elapsed={elapsed_min:.1f} min",
end="\r", flush=True)
if status == "complete":
@@ -441,7 +456,7 @@ def poll_kernel(api, kernel_ref: str, poll_interval: int, timeout_minutes: int)
f"https://www.kaggle.com/code/{kernel_ref}", "✗")
return False
if status == "cancelled":
if "cancel" in status:
print()
log("Kernel was cancelled.", "✗")
return False
@@ -515,8 +530,8 @@ def main():
# Import here so missing kaggle package gives a clean error
try:
from kaggle.api.kaggle_api_extended import KaggleApiExtended
api = KaggleApiExtended()
from kaggle.api.kaggle_api_extended import KaggleApi
api = KaggleApi()
api.authenticate()
except ImportError:
fatal("kaggle package not installed.\n Run: pip install kaggle")
View File
Whitespace-only changes.
+12
View File
@@ -0,0 +1,12 @@
"""
pip install hypercorn
"""
import asyncio
from hypercorn.config import Config
from hypercorn.asyncio import serve
from api_server import app
config = Config()
config.bind = ["0.0.0.0:2000"]
asyncio.run(serve(app, config))