feat: Initial project setup and core API functionality

This commit introduces the initial structure for the data platform, including:

- **Core Structure:** Setup of basic Go modules, environment, and project layers (cmd, internal, pkg).
- **SQLite Store:** Implements data persistence using SQLite, including schema initialization (collections/records).
- **CRUD Operations:** Full C.R.U.D. logic for records within collections.
- **Dynamic Querying:** Implements advanced query features over JSON data:
    - **Filtering:** Dynamic filters (eq, gt, lt, etc.) on JSON fields using SQL casting (`CAST(... AS REAL)`).
    - **Pagination:** Support for `limit` and `offset` query parameters.
    - **Sorting:** Dynamic sorting based on JSON fields (`orderBy=field` or `orderBy=-field`).
This commit is contained in:
Hadi Mottale 2025-11-03 13:42:53 +03:30
parent 3c9cba1018
commit 4704d7802b
7 changed files with 788 additions and 0 deletions

145
cmd/main.go Normal file
View File

@ -0,0 +1,145 @@
// cmd/main.go
package main
import (
"database/sql"
"fmt"
"log"
"net/http"
"time"
"upupa_dataist_ir/internal/api"
"upupa_dataist_ir/internal/store"
"upupa_dataist_ir/pkg/models"
"github.com/go-chi/chi/v5"
)
const DB_FILE = "upupa_dataist.db"
func main() {
log.Println("Initializing database...")
dbStore, err := store.NewSQLiteStore(DB_FILE)
if err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}
log.Println("Database connection successful.")
if err := dbStore.InitSchema(); err != nil {
log.Fatalf("Failed to initialize database schema: %v", err)
}
log.Println("Database schema initialized.")
if err := createSystemCollections(dbStore); err != nil {
log.Fatalf("Failed to create system collections: %v", err)
}
log.Println("System collections created successfully.")
if err := createProductCollection(dbStore, dbStore); err != nil {
log.Fatalf("Failed to create product records: %v", err)
}
log.Println("Product collection and test record created successfully.")
apiService := api.NewAPI(dbStore, dbStore)
router := chi.NewRouter()
router.Get("/api/health", apiService.HealthCheck)
router.Get("/api/collections/{name}/records/{id}", apiService.GetRecordByIDHandler)
router.Put("/api/collections/{name}/records/{id}", apiService.UpdateRecordHandler)
router.Delete("/api/collections/{name}/records/{id}", apiService.DeleteRecordHandler)
router.Get("/api/collections/{name}/records", apiService.GetAllRecordsHandler)
router.Post("/api/collections/{name}/records", apiService.CreateRecordHandler)
httpServer := &http.Server{
Addr: ":8081",
Handler: router,
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
}
log.Println("Server initialized. Starting API server on http://localhost:8081")
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("Could not start server: %v", err)
}
}
func createSystemCollections(repo store.CollectionRepository) error {
usersCollectionID := "r6k495b5i2n2x8k"
_, err := repo.GetByName("users")
if err == sql.ErrNoRows {
userCollection := models.Collection{
ID: usersCollectionID,
Name: "users",
System: true,
}
log.Println("Creating system collection 'users'...")
if _, err := repo.CreateCollection(userCollection); err != nil {
return err
}
} else if err != nil {
return err
}
return nil
}
func createProductCollection(colRepo store.CollectionRepository, recRepo store.RecordRepository) error {
productCollectionID := "t1t8t8p8c8c6v5s"
productID := "a4g7m2c9s3f1h0j"
_, err := colRepo.GetByName("products")
if err == sql.ErrNoRows {
productCollection := models.Collection{
ID: productCollectionID,
Name: "products",
System: false,
}
log.Println("Creating collection 'products'...")
if _, err := colRepo.CreateCollection(productCollection); err != nil {
return err
}
} else if err != nil {
return err
}
productData := map[string]interface{}{
"name": "تست",
"price": 49.99,
"stock": 100,
"releaseDate": time.Now().Format("2006-01-02"),
}
newRecord := models.Record{
ID: productID,
CollectionID: productCollectionID,
Data: productData,
}
log.Println("Adding test product record...")
if _, err := recRepo.CreateRecord(newRecord); err != nil {
if err.Error() == "UNIQUE constraint failed: records.id" ||
err.Error() == "UNIQUE constraint failed: records.collection_id, records.id" {
log.Println("Test record already exists, skipping creation.")
return nil
}
return fmt.Errorf("failed to create record: %w", err)
}
return nil
}

9
go.mod Normal file
View File

@ -0,0 +1,9 @@
module upupa_dataist_ir
go 1.24.6
require (
github.com/go-chi/chi/v5 v5.2.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mattn/go-sqlite3 v1.14.32 // indirect
)

6
go.sum Normal file
View File

@ -0,0 +1,6 @@
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=

251
internal/api/handlers.go Normal file
View File

@ -0,0 +1,251 @@
package api
import (
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"strconv"
"upupa_dataist_ir/internal/store"
"upupa_dataist_ir/pkg/models"
"github.com/go-chi/chi/v5"
)
type API struct {
Collections store.CollectionRepository
Records store.RecordRepository
}
func NewAPI(colRepo store.CollectionRepository, recRepo store.RecordRepository) *API {
return &API{
Collections: colRepo,
Records: recRepo,
}
}
func (a *API) HealthCheck(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("upupa_dataist_ir is running!"))
}
func (a *API) CreateRecordHandler(w http.ResponseWriter, r *http.Request) {
collectionName := chi.URLParam(r, "name")
collection, err := a.Collections.GetByName(collectionName)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound)
return
}
log.Printf("Error retrieving collection: %v", err)
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
var recordData map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&recordData); err != nil {
http.Error(w, "Invalid JSON payload", http.StatusBadRequest)
return
}
newRecord := models.Record{
ID: models.NewID(),
CollectionID: collection.ID,
Data: recordData,
}
createdRecord, err := a.Records.CreateRecord(newRecord)
if err != nil {
log.Printf("Error creating record: %v", err)
http.Error(w, "Failed to create record", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(createdRecord)
}
func (a *API) GetAllRecordsHandler(w http.ResponseWriter, r *http.Request) {
collectionName := chi.URLParam(r, "name")
collection, err := a.Collections.GetByName(collectionName)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound)
return
}
log.Printf("Error retrieving collection: %v", err)
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
queryParams := r.URL.Query()
filters := make(map[string]interface{})
for key, values := range queryParams {
if key != "limit" && key != "offset" && key != "orderBy" && len(values) > 0 {
filters[key] = values[0]
}
}
var finalFilters map[string]interface{} = nil
if len(filters) > 0 {
finalFilters = filters
}
limit := getIntQueryParam(queryParams.Get("limit"), 50)
offset := getIntQueryParam(queryParams.Get("offset"), 0)
orderBy := queryParams.Get("orderBy")
if limit < 0 || limit > 100 {
limit = 50
}
if offset < 0 {
offset = 0
}
records, err := a.Records.GetAll(collection.ID, finalFilters, orderBy, limit, offset)
if err != nil {
log.Printf("Error retrieving records: %v", err)
http.Error(w, "Failed to retrieve records", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
response := map[string]interface{}{
"items": records,
"totalItems": len(records),
"limit": limit,
"offset": offset,
"orderBy": orderBy,
}
json.NewEncoder(w).Encode(response)
}
func (a *API) GetRecordByIDHandler(w http.ResponseWriter, r *http.Request) {
collectionName := chi.URLParam(r, "name")
recordID := chi.URLParam(r, "id")
collection, err := a.Collections.GetByName(collectionName)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound)
return
}
log.Printf("Error retrieving collection: %v", err)
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
record, err := a.Records.GetByID(collection.ID, recordID)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound)
return
}
log.Printf("Error retrieving record: %v", err)
http.Error(w, "Failed to retrieve record", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(record)
}
func (a *API) UpdateRecordHandler(w http.ResponseWriter, r *http.Request) {
collectionName := chi.URLParam(r, "name")
recordID := chi.URLParam(r, "id")
collection, err := a.Collections.GetByName(collectionName)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound)
return
}
log.Printf("Error retrieving collection: %v", err)
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
var updatedData map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&updatedData); err != nil {
http.Error(w, "Invalid JSON payload", http.StatusBadRequest)
return
}
recordToUpdate := models.Record{
ID: recordID,
CollectionID: collection.ID,
Data: updatedData,
}
if err := a.Records.Update(recordToUpdate); err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound)
return
}
log.Printf("Error updating record: %v", err)
http.Error(w, "Failed to update record", http.StatusInternalServerError)
return
}
updatedRecord, err := a.Records.GetByID(collection.ID, recordID)
if err != nil {
log.Printf("Error fetching updated record: %v", err)
http.Error(w, "Record updated, but failed to fetch final data.", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(updatedRecord)
}
func (a *API) DeleteRecordHandler(w http.ResponseWriter, r *http.Request) {
collectionName := chi.URLParam(r, "name")
recordID := chi.URLParam(r, "id")
collection, err := a.Collections.GetByName(collectionName)
if err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Collection '%s' not found.", collectionName), http.StatusNotFound)
return
}
log.Printf("Error retrieving collection: %v", err)
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
if err := a.Records.Delete(collection.ID, recordID); err != nil {
if err == sql.ErrNoRows {
http.Error(w, fmt.Sprintf("Record with ID '%s' not found.", recordID), http.StatusNotFound)
return
}
log.Printf("Error deleting record: %v", err)
http.Error(w, "Failed to delete record", http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func getIntQueryParam(param string, defaultValue int) int {
if param == "" {
return defaultValue
}
val, err := strconv.Atoi(param)
if err != nil {
return defaultValue
}
return val
}

View File

@ -0,0 +1,19 @@
// internal/store/repository.go
package store
import "upupa_dataist_ir/pkg/models"
type CollectionRepository interface {
GetByName(name string) (models.Collection, error)
CreateCollection(c models.Collection) (models.Collection, error)
}
type RecordRepository interface {
CreateRecord(r models.Record) (models.Record, error)
GetAll(collectionID string, filters map[string]interface{}, orderBy string, limit, offset int) ([]models.Record, error)
GetByID(collectionID, recordID string) (models.Record, error)
Update(r models.Record) error
Delete(collectionID, recordID string) error
}

330
internal/store/sqlite.go Normal file
View File

@ -0,0 +1,330 @@
package store
import (
"database/sql"
"encoding/json"
"fmt"
"log"
"strings"
"upupa_dataist_ir/pkg/models"
_ "github.com/mattn/go-sqlite3"
)
type SQLiteStore struct {
db *sql.DB
}
func NewSQLiteStore(dbPath string) (*SQLiteStore, error) {
db, err := sql.Open("sqlite3", dbPath+"?_loc=UTC")
if err != nil {
return nil, err
}
if err := db.Ping(); err != nil {
return nil, err
}
return &SQLiteStore{db: db}, nil
}
func (s *SQLiteStore) InitSchema() error {
query := `
CREATE TABLE IF NOT EXISTS collections (
id TEXT PRIMARY KEY,
name TEXT UNIQUE NOT NULL,
system BOOLEAN DEFAULT FALSE
);
CREATE TABLE IF NOT EXISTS records (
id TEXT PRIMARY KEY,
collection_id TEXT NOT NULL,
created DATETIME DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
updated DATETIME DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
data TEXT NOT NULL,
FOREIGN KEY (collection_id) REFERENCES collections (id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_records_collection ON records (collection_id);
`
_, err := s.db.Exec(query)
return err
}
func (s *SQLiteStore) GetByName(name string) (models.Collection, error) {
query := "SELECT id, name, system FROM collections WHERE name = ?"
var c models.Collection
row := s.db.QueryRow(query, name)
err := row.Scan(&c.ID, &c.Name, &c.System)
if err == sql.ErrNoRows {
return c, err
}
return c, err
}
func (s *SQLiteStore) CreateCollection(c models.Collection) (models.Collection, error) {
query := `INSERT INTO collections (id, name, system) VALUES (?, ?, ?)`
_, err := s.db.Exec(query, c.ID, c.Name, c.System)
if err != nil {
return c, err
}
return c, nil
}
func (s *SQLiteStore) CreateRecord(r models.Record) (models.Record, error) {
jsonData, err := marshalRecordData(r.Data)
if err != nil {
return r, err
}
query := `INSERT INTO records
(id, collection_id, data)
VALUES (?, ?, ?)`
_, err = s.db.Exec(query, r.ID, r.CollectionID, jsonData)
if err != nil {
return r, err
}
return s.GetByID(r.CollectionID, r.ID)
}
func (s *SQLiteStore) GetAll(collectionID string, filters map[string]interface{}, orderBy string, limit, offset int) ([]models.Record, error) {
baseQuery := `SELECT id, collection_id, created, updated, data FROM records WHERE collection_id = ?`
args := []interface{}{collectionID}
if len(filters) > 0 {
filterClause := ""
for key, value := range filters {
field, op, err := parseFilterKey(key)
if err != nil {
log.Printf("Ignoring invalid filter key: %s, Error: %v", key, err)
continue
}
caster := getFieldCaster(field)
filterClause += fmt.Sprintf(" AND %s %s ?", caster, op)
args = append(args, value)
}
baseQuery += filterClause
}
sortClause := " ORDER BY created DESC"
if orderBy != "" {
sortClause = buildOrderClause(orderBy)
}
baseQuery += sortClause
if limit > 0 {
baseQuery += fmt.Sprintf(" LIMIT %d OFFSET %d", limit, offset)
}
rows, err := s.db.Query(baseQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
records := []models.Record{}
for rows.Next() {
var (
r models.Record
dataStr string
)
err := rows.Scan(&r.ID, &r.CollectionID, &r.Created, &r.Updated, &dataStr)
if err != nil {
return nil, err
}
data, err := unmarshalRecordData(dataStr)
if err != nil {
return nil, err
}
r.Data = data
records = append(records, r)
}
if err := rows.Err(); err != nil {
return nil, err
}
return records, nil
}
func (s *SQLiteStore) GetByID(collectionID, recordID string) (models.Record, error) {
query := `SELECT id, collection_id, created, updated, data
FROM records
WHERE collection_id = ? AND id = ?`
row := s.db.QueryRow(query, collectionID, recordID)
var (
r models.Record
dataStr string
)
err := row.Scan(&r.ID, &r.CollectionID, &r.Created, &r.Updated, &dataStr)
if err == sql.ErrNoRows {
return r, err
}
if err != nil {
return r, err
}
data, err := unmarshalRecordData(dataStr)
if err != nil {
return r, err
}
r.Data = data
return r, nil
}
func (s *SQLiteStore) Update(r models.Record) error {
jsonData, err := marshalRecordData(r.Data)
if err != nil {
return err
}
query := `UPDATE records SET data = ?, updated = (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) WHERE collection_id = ? AND id = ?`
result, err := s.db.Exec(query, jsonData, r.CollectionID, r.ID)
if err != nil {
return err
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return sql.ErrNoRows
}
return nil
}
func (s *SQLiteStore) Delete(collectionID, recordID string) error {
query := `DELETE FROM records WHERE collection_id = ? AND id = ?`
result, err := s.db.Exec(query, collectionID, recordID)
if err != nil {
return err
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return sql.ErrNoRows
}
return nil
}
func buildOrderClause(orderBy string) string {
orderBy = strings.TrimSpace(orderBy)
if orderBy == "" {
return " ORDER BY created DESC"
}
field := orderBy
direction := "ASC"
if strings.HasPrefix(orderBy, "-") {
field = orderBy[1:]
direction = "DESC"
}
caster := getFieldCaster(field)
return fmt.Sprintf(" ORDER BY %s %s", caster, direction)
}
func getFieldCaster(field string) string {
if field == "price" || field == "stock" {
return fmt.Sprintf("CAST(json_extract(data, '$.%s') AS REAL)", field)
}
if field == "releaseDate" {
return fmt.Sprintf("JULIANDAY(json_extract(data, '$.%s'))", field)
}
return fmt.Sprintf("json_extract(data, '$.%s')", field)
}
var validOperators = map[string]string{
"eq": "=",
"gt": ">",
"lt": "<",
"gte": ">=",
"lte": "<=",
"neq": "!=",
}
func parseFilterKey(key string) (field string, op string, err error) {
start := -1
end := -1
for i, r := range key {
if r == '[' {
start = i
} else if r == ']' {
end = i
break
}
}
if start == -1 || end == -1 || end <= start+1 || end != len(key)-1 {
return "", "", fmt.Errorf("invalid filter format: %s. Expected field[op]", key)
}
operatorCode := key[start+1 : end]
operator, ok := validOperators[operatorCode]
if !ok {
return "", "", fmt.Errorf("invalid operator: %s", operatorCode)
}
field = key[:start]
if field == "" {
return "", "", fmt.Errorf("field name cannot be empty")
}
return field, operator, nil
}
func marshalRecordData(data map[string]interface{}) (string, error) {
jsonData, err := json.Marshal(data)
if err != nil {
return "", fmt.Errorf("failed to marshal record data: %w", err)
}
return string(jsonData), nil
}
func unmarshalRecordData(dataStr string) (map[string]interface{}, error) {
var data map[string]interface{}
err := json.Unmarshal([]byte(dataStr), &data)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal record data: %w", err)
}
return data, nil
}

28
pkg/models/models.go Normal file
View File

@ -0,0 +1,28 @@
// pkg/models/models.go
package models
import (
"crypto/rand"
"encoding/hex"
"time"
)
type Collection struct {
ID string `json:"id"`
Name string `json:"name"`
System bool `json:"system"`
}
type Record struct {
ID string `json:"id"`
CollectionID string `json:"collectionId"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Data map[string]interface{} `json:"data"`
}
func NewID() string {
b := make([]byte, 6)
rand.Read(b)
return hex.EncodeToString(b)
}