From 60fed6d4641199e87d2ab35c99bb973d1565c1cb Mon Sep 17 00:00:00 2001 From: derekc Date: Wed, 18 Mar 2026 00:02:58 -0700 Subject: [PATCH] Implement performance improvements across backend and frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - models.py: add composite (user_id, date) indexes to flock_history, feed_purchases, and other_purchases for faster date-filtered queries (egg_collections already had one via its unique constraint) - main.py: add v2.2 migration to create the three composite indexes on existing installs at startup - stats.py: fix N+1 query in monthly_stats — flock history is now fetched once and looked up per month using bisect_right instead of one DB query per month row; also remove unnecessary Decimal(str(...)) round-trips since SQLAlchemy already returns Numeric columns as Decimal - eggs.py: add limit parameter (default 500, max 1000) to list_eggs to cap unbounded fetches on large datasets - dashboard.js: pass start= (30 days ago) when fetching eggs so the dashboard only loads the data it actually needs for the chart and recent collections list Co-Authored-By: Claude Sonnet 4.6 --- backend/main.py | 12 ++++++++++++ backend/models.py | 5 ++++- backend/routers/eggs.py | 3 +++ backend/routers/stats.py | 35 ++++++++++++++++++++--------------- nginx/html/js/dashboard.js | 9 +++++++-- 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/backend/main.py b/backend/main.py index 487a0b6..60615bf 100644 --- a/backend/main.py +++ b/backend/main.py @@ -79,6 +79,18 @@ def _run_migrations(): except Exception: db.rollback() # column already exists — safe to ignore + # v2.2 — composite (user_id, date) indexes for faster filtered queries + for sql in [ + "CREATE INDEX ix_flock_history_user_date ON flock_history (user_id, date)", + "CREATE INDEX ix_feed_purchases_user_date ON feed_purchases (user_id, date)", + "CREATE INDEX ix_other_purchases_user_date ON other_purchases (user_id, date)", + ]: + try: + db.execute(text(sql)) + db.commit() + except Exception: + db.rollback() # index already exists — safe to ignore + @asynccontextmanager async def lifespan(app: FastAPI): diff --git a/backend/models.py b/backend/models.py index 0ef807e..48020b1 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,5 +1,5 @@ from datetime import date, datetime -from sqlalchemy import Boolean, Integer, Date, DateTime, Text, Numeric, String, ForeignKey, UniqueConstraint, func +from sqlalchemy import Boolean, Integer, Date, DateTime, Text, Numeric, String, ForeignKey, UniqueConstraint, Index, func from sqlalchemy.orm import Mapped, mapped_column from database import Base @@ -30,6 +30,7 @@ class EggCollection(Base): class FlockHistory(Base): __tablename__ = "flock_history" + __table_args__ = (Index("ix_flock_history_user_date", "user_id", "date"),) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) @@ -41,6 +42,7 @@ class FlockHistory(Base): class FeedPurchase(Base): __tablename__ = "feed_purchases" + __table_args__ = (Index("ix_feed_purchases_user_date", "user_id", "date"),) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) @@ -53,6 +55,7 @@ class FeedPurchase(Base): class OtherPurchase(Base): __tablename__ = "other_purchases" + __table_args__ = (Index("ix_other_purchases_user_date", "user_id", "date"),) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) diff --git a/backend/routers/eggs.py b/backend/routers/eggs.py index 783188f..5832e10 100644 --- a/backend/routers/eggs.py +++ b/backend/routers/eggs.py @@ -17,13 +17,16 @@ router = APIRouter(prefix="/api/eggs", tags=["eggs"]) def list_eggs( start: Optional[date] = None, end: Optional[date] = None, + limit: int = 500, db: Session = Depends(get_db), current_user: User = Depends(get_current_user), ): + limit = min(limit, 1000) q = ( select(EggCollection) .where(EggCollection.user_id == current_user.id) .order_by(EggCollection.date.desc()) + .limit(limit) ) if start: q = q.where(EggCollection.date >= start) diff --git a/backend/routers/stats.py b/backend/routers/stats.py index 99c51ab..cb23d49 100644 --- a/backend/routers/stats.py +++ b/backend/routers/stats.py @@ -1,4 +1,5 @@ import calendar +from bisect import bisect_right from datetime import date, datetime, timedelta from decimal import Decimal from zoneinfo import ZoneInfo, ZoneInfoNotFoundError @@ -170,20 +171,24 @@ def monthly_stats( other_map = {(r.year, r.month): r.other_cost for r in other_rows} + # Fetch all flock history once (ascending) to avoid N+1 per month + flock_all = db.scalars( + select(FlockHistory) + .where(FlockHistory.user_id == uid) + .order_by(FlockHistory.date) + ).all() + flock_dates = [f.date for f in flock_all] + flock_counts = [f.chicken_count for f in flock_all] + + def flock_at(month_end: date) -> int | None: + idx = bisect_right(flock_dates, month_end) - 1 + return flock_counts[idx] if idx >= 0 else None + results = [] for row in egg_rows: y, m = int(row.year), int(row.month) - last_day = calendar.monthrange(y, m)[1] - month_end = date(y, m, last_day) - - flock_row = db.scalars( - select(FlockHistory) - .where(FlockHistory.user_id == uid) - .where(FlockHistory.date <= month_end) - .order_by(FlockHistory.date.desc()) - .limit(1) - ).first() - flock = flock_row.chicken_count if flock_row else None + month_end = date(y, m, calendar.monthrange(y, m)[1]) + flock = flock_at(month_end) total_eggs = int(row.total_eggs) days_logged = int(row.days_logged) @@ -192,11 +197,11 @@ def monthly_stats( raw_feed_cost = feed_map.get((y, m)) raw_other_cost = other_map.get((y, m)) - feed_cost = round(Decimal(str(raw_feed_cost)), 2) if raw_feed_cost else None - other_cost = round(Decimal(str(raw_other_cost)), 2) if raw_other_cost else None + feed_cost = round(raw_feed_cost, 2) if raw_feed_cost else None + other_cost = round(raw_other_cost, 2) if raw_other_cost else None - raw_total_cost = (raw_feed_cost or 0) + (raw_other_cost or 0) - cpe = round(Decimal(str(raw_total_cost)) / Decimal(total_eggs), 4) if (raw_total_cost and total_eggs) else None + total_cost = (raw_feed_cost or Decimal(0)) + (raw_other_cost or Decimal(0)) + cpe = round(total_cost / total_eggs, 4) if (total_cost and total_eggs) else None cpd = round(cpe * 12, 4) if cpe else None results.append(MonthlySummary( diff --git a/nginx/html/js/dashboard.js b/nginx/html/js/dashboard.js index 1a0ee4c..6ffd148 100644 --- a/nginx/html/js/dashboard.js +++ b/nginx/html/js/dashboard.js @@ -80,11 +80,16 @@ async function loadDashboard() { const msg = document.getElementById('msg'); try { - // Fetch stats and recent eggs in parallel + // Fetch stats and recent eggs in parallel; eggs limited to last 30 days for chart + recent list + const tz = (typeof Auth !== 'undefined' && Auth.getUser()?.timezone) || 'UTC'; + const start30 = new Date(); + start30.setDate(start30.getDate() - 30); + const start30str = start30.toLocaleDateString('en-CA', { timeZone: tz }); + const [stats, budget, eggs] = await Promise.all([ API.get('/api/stats/dashboard'), API.get('/api/stats/budget'), - API.get('/api/eggs'), + API.get(`/api/eggs?start=${start30str}`), ]); // Populate stat cards