From 4704d7802b8596eebdd40c8603029c0aaf582855 Mon Sep 17 00:00:00 2001 From: Hadi Mottale Date: Mon, 3 Nov 2025 13:42:53 +0330 Subject: [PATCH] feat: Initial project setup and core API functionality This commit introduces the initial structure for the data platform, including: - **Core Structure:** Setup of basic Go modules, environment, and project layers (cmd, internal, pkg). - **SQLite Store:** Implements data persistence using SQLite, including schema initialization (collections/records). - **CRUD Operations:** Full C.R.U.D. logic for records within collections. - **Dynamic Querying:** Implements advanced query features over JSON data: - **Filtering:** Dynamic filters (eq, gt, lt, etc.) on JSON fields using SQL casting (`CAST(... AS REAL)`). - **Pagination:** Support for `limit` and `offset` query parameters. - **Sorting:** Dynamic sorting based on JSON fields (`orderBy=field` or `orderBy=-field`). --- cmd/main.go | 145 +++++++++++++++ go.mod | 9 + go.sum | 6 + internal/api/handlers.go | 251 ++++++++++++++++++++++++++ internal/store/repository.go | 19 ++ internal/store/sqlite.go | 330 +++++++++++++++++++++++++++++++++++ pkg/models/models.go | 28 +++ 7 files changed, 788 insertions(+) create mode 100644 cmd/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/api/handlers.go create mode 100644 internal/store/repository.go create mode 100644 internal/store/sqlite.go create mode 100644 pkg/models/models.go diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..fde8b80 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,145 @@ +// cmd/main.go +package main + +import ( + "database/sql" + "fmt" + "log" + "net/http" + "time" + + "upupa_dataist_ir/internal/api" + "upupa_dataist_ir/internal/store" + "upupa_dataist_ir/pkg/models" + + "github.com/go-chi/chi/v5" +) + +const DB_FILE = "upupa_dataist.db" + +func main() { + log.Println("Initializing database...") + + dbStore, err := store.NewSQLiteStore(DB_FILE) + if err != nil { + log.Fatalf("Failed to connect to database: %v", err) + } + log.Println("Database connection successful.") + + if err := dbStore.InitSchema(); err != nil { + log.Fatalf("Failed to initialize database schema: %v", err) + } + log.Println("Database schema initialized.") + + if err := createSystemCollections(dbStore); err != nil { + log.Fatalf("Failed to create system collections: %v", err) + } + log.Println("System collections created successfully.") + + if err := createProductCollection(dbStore, dbStore); err != nil { + log.Fatalf("Failed to create product records: %v", err) + } + log.Println("Product collection and test record created successfully.") + + apiService := api.NewAPI(dbStore, dbStore) + router := chi.NewRouter() + + router.Get("/api/health", apiService.HealthCheck) + + router.Get("/api/collections/{name}/records/{id}", apiService.GetRecordByIDHandler) + router.Put("/api/collections/{name}/records/{id}", apiService.UpdateRecordHandler) + router.Delete("/api/collections/{name}/records/{id}", apiService.DeleteRecordHandler) + + router.Get("/api/collections/{name}/records", apiService.GetAllRecordsHandler) + router.Post("/api/collections/{name}/records", apiService.CreateRecordHandler) + + httpServer := &http.Server{ + Addr: ":8081", + Handler: router, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 120 * time.Second, + } + + log.Println("Server initialized. Starting API server on http://localhost:8081") + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("Could not start server: %v", err) + } + +} + +func createSystemCollections(repo store.CollectionRepository) error { + usersCollectionID := "r6k495b5i2n2x8k" + + _, err := repo.GetByName("users") + + if err == sql.ErrNoRows { + + userCollection := models.Collection{ + ID: usersCollectionID, + Name: "users", + System: true, + } + + log.Println("Creating system collection 'users'...") + if _, err := repo.CreateCollection(userCollection); err != nil { + return err + } + + } else if err != nil { + return err + } + + return nil +} + +func createProductCollection(colRepo store.CollectionRepository, recRepo store.RecordRepository) error { + productCollectionID := "t1t8t8p8c8c6v5s" + productID := "a4g7m2c9s3f1h0j" + + _, err := colRepo.GetByName("products") + + if err == sql.ErrNoRows { + + productCollection := models.Collection{ + ID: productCollectionID, + Name: "products", + System: false, + } + + log.Println("Creating collection 'products'...") + if _, err := colRepo.CreateCollection(productCollection); err != nil { + return err + } + + } else if err != nil { + return err + } + + productData := map[string]interface{}{ + "name": "تست", + "price": 49.99, + "stock": 100, + "releaseDate": time.Now().Format("2006-01-02"), + } + + newRecord := models.Record{ + ID: productID, + CollectionID: productCollectionID, + Data: productData, + } + + log.Println("Adding test product record...") + + if _, err := recRepo.CreateRecord(newRecord); err != nil { + + if err.Error() == "UNIQUE constraint failed: records.id" || + err.Error() == "UNIQUE constraint failed: records.collection_id, records.id" { + log.Println("Test record already exists, skipping creation.") + return nil + } + return fmt.Errorf("failed to create record: %w", err) + } + + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..048e468 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module upupa_dataist_ir + +go 1.24.6 + +require ( + github.com/go-chi/chi/v5 v5.2.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..df4eb1f --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= diff --git a/internal/api/handlers.go b/internal/api/handlers.go new file mode 100644 index 0000000..f04b684 --- /dev/null +++ b/internal/api/handlers.go @@ -0,0 +1,251 @@ +package api + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + "upupa_dataist_ir/internal/store" + "upupa_dataist_ir/pkg/models" + + "github.com/go-chi/chi/v5" +) + +type API struct { + Collections store.CollectionRepository + Records store.RecordRepository +} + +func NewAPI(colRepo store.CollectionRepository, recRepo store.RecordRepository) *API { + return &API{ + Collections: colRepo, + Records: recRepo, + } +} + +func (a *API) HealthCheck(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("upupa_dataist_ir is running!")) +} + +func (a *API) CreateRecordHandler(w http.ResponseWriter, r *http.Request) { + collectionName := chi.URLParam(r, "name") + + collection, err := a.Collections.GetByName(collectionName) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound) + return + } + log.Printf("Error retrieving collection: %v", err) + http.Error(w, "Database error", http.StatusInternalServerError) + return + } + + var recordData map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&recordData); err != nil { + http.Error(w, "Invalid JSON payload", http.StatusBadRequest) + return + } + + newRecord := models.Record{ + ID: models.NewID(), + CollectionID: collection.ID, + Data: recordData, + } + + createdRecord, err := a.Records.CreateRecord(newRecord) + if err != nil { + log.Printf("Error creating record: %v", err) + http.Error(w, "Failed to create record", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + + json.NewEncoder(w).Encode(createdRecord) +} + +func (a *API) GetAllRecordsHandler(w http.ResponseWriter, r *http.Request) { + collectionName := chi.URLParam(r, "name") + + collection, err := a.Collections.GetByName(collectionName) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound) + return + } + log.Printf("Error retrieving collection: %v", err) + http.Error(w, "Database error", http.StatusInternalServerError) + return + } + + queryParams := r.URL.Query() + filters := make(map[string]interface{}) + + for key, values := range queryParams { + if key != "limit" && key != "offset" && key != "orderBy" && len(values) > 0 { + filters[key] = values[0] + } + } + + var finalFilters map[string]interface{} = nil + if len(filters) > 0 { + finalFilters = filters + } + + limit := getIntQueryParam(queryParams.Get("limit"), 50) + offset := getIntQueryParam(queryParams.Get("offset"), 0) + orderBy := queryParams.Get("orderBy") + + if limit < 0 || limit > 100 { + limit = 50 + } + if offset < 0 { + offset = 0 + } + + records, err := a.Records.GetAll(collection.ID, finalFilters, orderBy, limit, offset) + if err != nil { + log.Printf("Error retrieving records: %v", err) + http.Error(w, "Failed to retrieve records", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + response := map[string]interface{}{ + "items": records, + "totalItems": len(records), + "limit": limit, + "offset": offset, + "orderBy": orderBy, + } + + json.NewEncoder(w).Encode(response) +} + +func (a *API) GetRecordByIDHandler(w http.ResponseWriter, r *http.Request) { + collectionName := chi.URLParam(r, "name") + recordID := chi.URLParam(r, "id") + + collection, err := a.Collections.GetByName(collectionName) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound) + return + } + log.Printf("Error retrieving collection: %v", err) + http.Error(w, "Database error", http.StatusInternalServerError) + return + } + + record, err := a.Records.GetByID(collection.ID, recordID) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound) + return + } + log.Printf("Error retrieving record: %v", err) + http.Error(w, "Failed to retrieve record", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + json.NewEncoder(w).Encode(record) +} + +func (a *API) UpdateRecordHandler(w http.ResponseWriter, r *http.Request) { + collectionName := chi.URLParam(r, "name") + recordID := chi.URLParam(r, "id") + + collection, err := a.Collections.GetByName(collectionName) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound) + return + } + log.Printf("Error retrieving collection: %v", err) + http.Error(w, "Database error", http.StatusInternalServerError) + return + } + + var updatedData map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&updatedData); err != nil { + http.Error(w, "Invalid JSON payload", http.StatusBadRequest) + return + } + + recordToUpdate := models.Record{ + ID: recordID, + CollectionID: collection.ID, + Data: updatedData, + } + + if err := a.Records.Update(recordToUpdate); err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound) + return + } + log.Printf("Error updating record: %v", err) + http.Error(w, "Failed to update record", http.StatusInternalServerError) + return + } + + updatedRecord, err := a.Records.GetByID(collection.ID, recordID) + if err != nil { + log.Printf("Error fetching updated record: %v", err) + http.Error(w, "Record updated, but failed to fetch final data.", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + json.NewEncoder(w).Encode(updatedRecord) +} + +func (a *API) DeleteRecordHandler(w http.ResponseWriter, r *http.Request) { + collectionName := chi.URLParam(r, "name") + recordID := chi.URLParam(r, "id") + + collection, err := a.Collections.GetByName(collectionName) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound) + return + } + log.Printf("Error retrieving collection: %v", err) + http.Error(w, "Database error", http.StatusInternalServerError) + return + } + + if err := a.Records.Delete(collection.ID, recordID); err != nil { + if err == sql.ErrNoRows { + http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound) + return + } + log.Printf("Error deleting record: %v", err) + http.Error(w, "Failed to delete record", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func getIntQueryParam(param string, defaultValue int) int { + if param == "" { + return defaultValue + } + val, err := strconv.Atoi(param) + if err != nil { + return defaultValue + } + return val +} diff --git a/internal/store/repository.go b/internal/store/repository.go new file mode 100644 index 0000000..d0c1a5b --- /dev/null +++ b/internal/store/repository.go @@ -0,0 +1,19 @@ +// internal/store/repository.go +package store + +import "upupa_dataist_ir/pkg/models" + +type CollectionRepository interface { + GetByName(name string) (models.Collection, error) + CreateCollection(c models.Collection) (models.Collection, error) +} + +type RecordRepository interface { + CreateRecord(r models.Record) (models.Record, error) + + GetAll(collectionID string, filters map[string]interface{}, orderBy string, limit, offset int) ([]models.Record, error) + + GetByID(collectionID, recordID string) (models.Record, error) + Update(r models.Record) error + Delete(collectionID, recordID string) error +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go new file mode 100644 index 0000000..ad9b521 --- /dev/null +++ b/internal/store/sqlite.go @@ -0,0 +1,330 @@ +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "strings" + "upupa_dataist_ir/pkg/models" + + _ "github.com/mattn/go-sqlite3" +) + +type SQLiteStore struct { + db *sql.DB +} + +func NewSQLiteStore(dbPath string) (*SQLiteStore, error) { + db, err := sql.Open("sqlite3", dbPath+"?_loc=UTC") + if err != nil { + return nil, err + } + + if err := db.Ping(); err != nil { + return nil, err + } + + return &SQLiteStore{db: db}, nil +} + +func (s *SQLiteStore) InitSchema() error { + query := ` + CREATE TABLE IF NOT EXISTS collections ( + id TEXT PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + system BOOLEAN DEFAULT FALSE + ); + + CREATE TABLE IF NOT EXISTS records ( + id TEXT PRIMARY KEY, + collection_id TEXT NOT NULL, + created DATETIME DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + updated DATETIME DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + data TEXT NOT NULL, + FOREIGN KEY (collection_id) REFERENCES collections (id) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_records_collection ON records (collection_id); + ` + + _, err := s.db.Exec(query) + return err +} + +func (s *SQLiteStore) GetByName(name string) (models.Collection, error) { + query := "SELECT id, name, system FROM collections WHERE name = ?" + + var c models.Collection + row := s.db.QueryRow(query, name) + + err := row.Scan(&c.ID, &c.Name, &c.System) + + if err == sql.ErrNoRows { + return c, err + } + + return c, err +} + +func (s *SQLiteStore) CreateCollection(c models.Collection) (models.Collection, error) { + query := `INSERT INTO collections (id, name, system) VALUES (?, ?, ?)` + + _, err := s.db.Exec(query, c.ID, c.Name, c.System) + if err != nil { + return c, err + } + + return c, nil +} + +func (s *SQLiteStore) CreateRecord(r models.Record) (models.Record, error) { + jsonData, err := marshalRecordData(r.Data) + if err != nil { + return r, err + } + + query := `INSERT INTO records + (id, collection_id, data) + VALUES (?, ?, ?)` + + _, err = s.db.Exec(query, r.ID, r.CollectionID, jsonData) + if err != nil { + return r, err + } + + return s.GetByID(r.CollectionID, r.ID) +} + +func (s *SQLiteStore) GetAll(collectionID string, filters map[string]interface{}, orderBy string, limit, offset int) ([]models.Record, error) { + + baseQuery := `SELECT id, collection_id, created, updated, data FROM records WHERE collection_id = ?` + + args := []interface{}{collectionID} + + if len(filters) > 0 { + filterClause := "" + + for key, value := range filters { + field, op, err := parseFilterKey(key) + if err != nil { + log.Printf("Ignoring invalid filter key: %s, Error: %v", key, err) + continue + } + + caster := getFieldCaster(field) + + filterClause += fmt.Sprintf(" AND %s %s ?", caster, op) + args = append(args, value) + } + + baseQuery += filterClause + } + + sortClause := " ORDER BY created DESC" + if orderBy != "" { + sortClause = buildOrderClause(orderBy) + } + baseQuery += sortClause + + if limit > 0 { + baseQuery += fmt.Sprintf(" LIMIT %d OFFSET %d", limit, offset) + } + + rows, err := s.db.Query(baseQuery, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + records := []models.Record{} + + for rows.Next() { + var ( + r models.Record + dataStr string + ) + + err := rows.Scan(&r.ID, &r.CollectionID, &r.Created, &r.Updated, &dataStr) + if err != nil { + return nil, err + } + + data, err := unmarshalRecordData(dataStr) + if err != nil { + return nil, err + } + + r.Data = data + records = append(records, r) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return records, nil +} + +func (s *SQLiteStore) GetByID(collectionID, recordID string) (models.Record, error) { + query := `SELECT id, collection_id, created, updated, data + FROM records + WHERE collection_id = ? AND id = ?` + + row := s.db.QueryRow(query, collectionID, recordID) + + var ( + r models.Record + dataStr string + ) + + err := row.Scan(&r.ID, &r.CollectionID, &r.Created, &r.Updated, &dataStr) + + if err == sql.ErrNoRows { + return r, err + } + if err != nil { + return r, err + } + + data, err := unmarshalRecordData(dataStr) + if err != nil { + return r, err + } + + r.Data = data + + return r, nil +} + +func (s *SQLiteStore) Update(r models.Record) error { + jsonData, err := marshalRecordData(r.Data) + if err != nil { + return err + } + + query := `UPDATE records SET data = ?, updated = (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) WHERE collection_id = ? AND id = ?` + + result, err := s.db.Exec(query, jsonData, r.CollectionID, r.ID) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + + if rowsAffected == 0 { + return sql.ErrNoRows + } + + return nil +} + +func (s *SQLiteStore) Delete(collectionID, recordID string) error { + query := `DELETE FROM records WHERE collection_id = ? AND id = ?` + + result, err := s.db.Exec(query, collectionID, recordID) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + + if rowsAffected == 0 { + return sql.ErrNoRows + } + + return nil +} + +func buildOrderClause(orderBy string) string { + orderBy = strings.TrimSpace(orderBy) + if orderBy == "" { + return " ORDER BY created DESC" + } + + field := orderBy + direction := "ASC" + + if strings.HasPrefix(orderBy, "-") { + field = orderBy[1:] + direction = "DESC" + } + + caster := getFieldCaster(field) + + return fmt.Sprintf(" ORDER BY %s %s", caster, direction) +} + +func getFieldCaster(field string) string { + if field == "price" || field == "stock" { + return fmt.Sprintf("CAST(json_extract(data, '$.%s') AS REAL)", field) + } + + if field == "releaseDate" { + return fmt.Sprintf("JULIANDAY(json_extract(data, '$.%s'))", field) + } + + return fmt.Sprintf("json_extract(data, '$.%s')", field) +} + +var validOperators = map[string]string{ + "eq": "=", + "gt": ">", + "lt": "<", + "gte": ">=", + "lte": "<=", + "neq": "!=", +} + +func parseFilterKey(key string) (field string, op string, err error) { + start := -1 + end := -1 + for i, r := range key { + if r == '[' { + start = i + } else if r == ']' { + end = i + break + } + } + + if start == -1 || end == -1 || end <= start+1 || end != len(key)-1 { + return "", "", fmt.Errorf("invalid filter format: %s. Expected field[op]", key) + } + + operatorCode := key[start+1 : end] + operator, ok := validOperators[operatorCode] + if !ok { + return "", "", fmt.Errorf("invalid operator: %s", operatorCode) + } + + field = key[:start] + if field == "" { + return "", "", fmt.Errorf("field name cannot be empty") + } + + return field, operator, nil +} + +func marshalRecordData(data map[string]interface{}) (string, error) { + jsonData, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("failed to marshal record data: %w", err) + } + return string(jsonData), nil +} + +func unmarshalRecordData(dataStr string) (map[string]interface{}, error) { + var data map[string]interface{} + err := json.Unmarshal([]byte(dataStr), &data) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal record data: %w", err) + } + return data, nil +} diff --git a/pkg/models/models.go b/pkg/models/models.go new file mode 100644 index 0000000..07592de --- /dev/null +++ b/pkg/models/models.go @@ -0,0 +1,28 @@ +// pkg/models/models.go +package models + +import ( + "crypto/rand" + "encoding/hex" + "time" +) + +type Collection struct { + ID string `json:"id"` + Name string `json:"name"` + System bool `json:"system"` +} + +type Record struct { + ID string `json:"id"` + CollectionID string `json:"collectionId"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Data map[string]interface{} `json:"data"` +} + +func NewID() string { + b := make([]byte, 6) + rand.Read(b) + return hex.EncodeToString(b) +}