Feature/fly io setup

.air.toml

@@ -0,0 +1,20 @@
+root = "."
+tmp_dir = "tmp"
+
+[build]
+cmd = "go build -o ./tmp/main.exe ./src"
+bin = "tmp\\main.exe"
+include_ext = ["go"]
+exclude_dir = ["tmp", "vendor", "testdata"]
+exclude_regex = ["_test\\.go$"]
+delay = 1000
+
+[log]
+time = true
+main_only = false
+
+[screen]
+clear_on_rebuild = true
+
+[misc]
+clean_on_exit = true

.dockerignore

@@ -0,0 +1,8 @@
+.git
+.env
+.env.*
+tmp/
+.obsidian/
+docs/
+*.md
+!README.md 

.env.example

@@ -1,10 +1,10 @@
 # App
-APP_ENV=development
-PORT=8080
+APP_ENV=development  # development or production
+PORT=8080           # API server port
 
-# Turso
-DATABASE_URL=libsql://your-db-name.turso.io
-DATABASE_AUTH_TOKEN=your_auth_token
+# Turso Database
+DATABASE_URL=libsql://your-db-name.turso.io      # Your Turso database URL
+DATABASE_AUTH_TOKEN=your_auth_token               # Your Turso auth token
 
 # Clerk
 CLERK_PUBLISHABLE_KEY=pk_test_...
@@ -18,6 +18,7 @@ PADDLE_PUBLIC_KEY=your_public_key
 # Fly.io
 FLY_API_TOKEN=your_fly_token
 
-# Optional: For development
-DEBUG=true
-LOG_LEVEL=debug
+# Development
+DEBUG=true                  # Enable debug logging
+LOG_LEVEL=debug            # debug, info, warn, or error
+RUN_INTEGRATION_TESTS=true # Enable integration tests

CONTRIBUTING.md

@@ -0,0 +1,20 @@
+# Contributing to Cache Warmer
+
+## Development Setup
+
+1. Fork and clone the repository
+2. Copy `.env.example` to `.env` and configure
+3. Run tests with `go test ./... -v`
+
+## Testing
+
+- Unit tests: `go test ./...`
+- Integration tests: Require Turso credentials in `.env`
+- Test coverage: `go test ./... -cover`
+
+## Pull Request Process
+
+1. Update documentation
+2. Add/update tests
+3. Ensure all tests pass
+4. Update the development plan if needed

Dockerfile

@@ -0,0 +1,30 @@
+# Build stage
+FROM golang:1.21-alpine AS builder
+
+WORKDIR /app
+
+# Copy go mod and sum files
+COPY go.mod go.sum ./
+
+# Download dependencies
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the application
+RUN CGO_ENABLED=0 GOOS=linux go build -o main ./src/main.go
+
+# Final stage
+FROM alpine:latest
+
+WORKDIR /app
+
+# Copy binary from builder
+COPY --from=builder /app/main .
+
+# Expose port
+EXPOSE 8080
+
+# Run the binary
+CMD ["./main"]

README.md

@@ -10,20 +10,64 @@ Cache Warmer crawls your Webflow site after publishing, ensuring all pages are c
 
 🚧 Currently in initial development
 
+Core functionality implemented:
+
+- ✅ URL crawling with concurrent requests
+- ✅ Database integration with Turso
+- ✅ Basic error handling
+- ✅ Test coverage for core components
+
+Next up:
+
+- 🔄 Rate limiting and retry logic
+- 🔄 Fly.io deployment
+- 🔄 Cache validation improvements
+
 ## Tech Stack
 
 - Backend: Go
 - Database: Turso
-- Hosting: Fly.io
-- Auth: Clerk
-- Payments: Paddle
-- Frontend: Webflow
+- Hosting: Fly.io (coming soon)
+- Auth: Clerk (planned)
+- Payments: Paddle (planned)
+- Frontend: Webflow (planned)
 
 ## Local Development
 
-[Coming soon - will include setup instructions]
+### Prerequisites
+
+- Go 1.23 or later
+- A Turso database account
+- Git
+
+### Setup
+
+1. Clone the repository:
+
+   ```bash
+   git clone https://github.com/teamharvey/cache-warmer.git
+   cd cache-warmer
+   ```
+
+2. Set up environment:
+
+   ```bash
+   cp .env.example .env
+   # Edit .env with your Turso credentials
+   ```
+
+3. Install dependencies:
+
+   ```bash
+   go mod download
+   ```
+
+4. Run tests:
+   ```bash
+   go test ./... -v
+   ```
 
-## Project Structure
+### Project Structure
 
 ## Environment Setup
 

Planning/Development plan.md → docs/planning/Development plan.md

@@ -1,50 +1,63 @@
-## Stage 0: Project Setup & Infrastructure (6-10 hrs)
-
-### Development Environment Setup (2-3 hrs)
-
-- [ ] Initialise GitHub repository
-- [ ] Set up branch protection
-- [ ] Create dev/prod branches
-- [ ] Set up local development environment
-- [ ] Add initial documentation
-### Go Project Structure (2-3 hrs)
-
-- [ ] Initialize Go project
-- [ ] Set up dependency management
-- [ ] Create project structure
-- [ ] Add basic configs
-- [ ] Set up testing framework
-### Environment Configuration (2-4 hrs)
-
-- [ ] Set up dev/prod environments
-- [ ] Configure environment variables
-- [ ] Set up secrets management
-- [ ] Configure Fly.io for both environments
-- [ ] Set up initial deployment workflow
-## Stage 1: Core Setup & Basic Crawling (15-25 hrs)
-
-### Set up Go project with Fly.io deployment (3-5 hrs)
-
-- [ ] Initialize Go project structure and dependencies
-- [ ] Set up basic API endpoints
-- [ ] Configure Fly.io deployment settings
-- [ ] Set up environment variables and configs
-- [ ] Implement basic health checks and monitoring
-### Implement basic crawler (using Go's Colly) (8-12 hrs)
-
-- [ ] Set up Colly crawler configuration
-- [ ] Implement concurrent crawling logic
+## Stage 0: Project Setup & Infrastructure (6-10 hrs) 🟡
+
+### Development Environment Setup (2-3 hrs) ✅
+
+- [x] Initialise GitHub repository
+- [x] Set up branch protection
+- [x] Create dev/prod branches
+- [x] Set up local development environment
+- [x] Add initial documentation
+
+### Go Project Structure (2-3 hrs) ✅
+
+- [x] Initialize Go project
+- [x] Set up dependency management
+- [x] Create project structure
+- [x] Add basic configs
+- [x] Set up testing framework
+
+### Production Infrastructure Setup (2-4 hrs) ⚡Priority
+
+- [x] Set up dev/prod environments
+- [x] Configure environment variables
+- [x] Set up secrets management
+- [ ] Configure Fly.io
+  - [ ] Set up Fly.io account and project
+  - [ ] Configure deployment settings
+  - [ ] Set up environment variables in Fly.io
+  - [ ] Create deployment workflow
+  - [ ] Add health check endpoint monitoring
+- [ ] Test production deployment
+
+## Stage 1: Core Setup & Basic Crawling (15-25 hrs) 🟡
+
+### Core API Implementation (3-5 hrs)
+
+- [x] Initialize Go project structure and dependencies
+- [x] Set up basic API endpoints
+- [x] Set up environment variables and configs
+- [x] Implement basic health checks and monitoring
+
+### Enhance Crawler Results (8-12 hrs) ⚡Priority
+
+- [x] Set up Colly crawler configuration
+- [x] Implement concurrent crawling logic
+- [x] Add basic error handling
 - [ ] Add rate limiting and retry logic
 - [ ] Handle different response types/errors
 - [ ] Implement cache validation checks
 - [ ] Add performance metrics collection
-### Set up Turso for storing results (4-8 hrs)
 
-- [ ] Design database schema
-- [ ] Set up Turso connection and config
-- [ ] Implement data models and queries
-- [ ] Add error handling and retries
+### Set up Turso for storing results (4-8 hrs) 🟡
+
+- [x] Design database schema
+- [x] Set up Turso connection and config
+- [x] Implement data models and queries
+- [x] Add basic error handling
+- [x] Set up integration tests
+- [ ] Add retry logic
 - [ ] Set up basic data cleanup routines
+
 ## Stage 2: Auth & User Management (10-16 hrs)
 
 ### Implement Clerk authentication (4-6 hrs)
@@ -61,13 +74,15 @@
 - [ ] Implement user profile storage
 - [ ] Add user preferences handling
 - [ ] Set up user data sync with Clerk
+
 ### Set up basic usage tracking (4-6 hrs)
 
 - [ ] Implement usage counters
 - [ ] Add usage limits checking
 - [ ] Set up usage reset schedule
 - [ ] Implement usage notifications
 - [ ] Add basic reporting functions
+
 ## Stage 3: Billing & Subscriptions (8-12 hrs)
 
 ### Implement Paddle integration (4-6 hrs)
@@ -76,19 +91,22 @@
 - [ ] Implement subscription webhooks
 - [ ] Add payment flow integration
 - [ ] Set up subscription plans
-- [ ] Implement checkout process 
+- [ ] Implement checkout process
+
 ### Connect subscription status to user accounts (2-3 hrs)
 
 - [ ] Link subscriptions to users
 - [ ] Handle subscription updates
 - [ ] Implement plan changes
 - [ ] Add subscription status checks
+
 ### Add usage limits/tracking (2-3 hrs)
 
 - [ ] Implement plan-based limits
 - [ ] Add upgrade prompts
 - [ ] Set up usage warnings
-- [ ] Implement grace period    
+- [ ] Implement grace period
+
 ## Stage 4: Webflow Integration & Launch (8-16 hrs)
 
 ### Build Webflow frontend interface (4-8 hrs)
@@ -98,25 +116,28 @@
 - [ ] Implement results display
 - [ ] Add usage statistics display
 - [ ] Create settings interface
+
 ### Connect to backend APIs (3-5 hrs)
 
 - [ ] Implement API calls
 - [ ] Add error handling
 - [ ] Set up response handling
 - [ ] Implement loading states
 - [ ] Add retry logic
+
 ### Set up monitoring (GA) (1-3 hrs)
 
 - [ ] Configure GA tracking
 - [ ] Add custom events
 - [ ] Set up conversion tracking
 - [ ] Implement error tracking
-- [ ] Create basic dashboards    
+- [ ] Create basic dashboards
 
 ---
+
 ## Key Risk Areas:
 
 - [ ] Crawler edge cases and error handling
 - [ ] Auth integration complexity
 - [ ] Paddle webhook handling
-- [ ] Webflow API limitations
+- [ ] Webflow API limitations

fly.toml

@@ -0,0 +1,25 @@
+app = "blue-banded-bee"
+primary_region = "syd"  # Sydney region, since we're in AU
+
+[build]
+  dockerfile = "Dockerfile"
+
+[env]
+  PORT = "8080"
+  APP_ENV = "production"
+
+[http_service]
+  internal_port = 8080
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 1
+  processes = ["app"]
+
+[[http_service.checks]]
+  grace_period = "10s"
+  interval = "30s"
+  method = "GET"
+  path = "/health"
+  protocol = "http"
+  timeout = "5s"