summaryrefslogtreecommitdiff
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/build-and-deploy-prod.yml2
-rw-r--r--.github/workflows/build-and-deploy-stage.yml2
-rw-r--r--config/config.go86
-rw-r--r--config/config.local.yaml28
-rw-r--r--config/config.prod.yaml30
-rw-r--r--config/config.stage.yaml29
-rw-r--r--deploy/inventory.yml2
-rw-r--r--deploy/playbook.yml28
-rw-r--r--go.mod1
-rw-r--r--go.sum61
-rw-r--r--internal/blog/index.go32
-rw-r--r--internal/blog/s3.go173
-rw-r--r--internal/router/handlers/root.go3
-rw-r--r--internal/router/router.go36
-rw-r--r--main.go2
-rw-r--r--static/input.css2
-rw-r--r--views/layouts/general-page.html3
-rw-r--r--views/pages/global-map.html5
-rw-r--r--views/pages/robots.txt4
19 files changed, 394 insertions, 135 deletions
diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml
index d89cc76..97afdf7 100644
--- a/.github/workflows/build-and-deploy-prod.yml
+++ b/.github/workflows/build-and-deploy-prod.yml
@@ -80,4 +80,4 @@ jobs:
ANSIBLE_HOST_KEY_CHECKING: False
working-directory: ./deploy
run: |
- ansible-playbook -i inventory.yml -l prod playbook.yml --private-key ~/.ssh/id_ed25519 -u svc_github -e saya_today_web_auth_salt="${{ secrets.AUTH_SALT }}" -e saya_today_web_b2_key_id="${{ secrets.B2_KEY_ID }}" -e saya_today_web_b2_application_key="${{ secrets.B2_APPLICATION_KEY }}" -e saya_today_web_environment=prod -e saya_today_web_tag=${{ github.ref_name }} -e saya_today_web_mail_salt="${{ secrets.MAIL_SALT }}" -e saya_today_web_mail_host="${{ secrets.MAIL_HOST }}" -e saya_today_web_mail_address="${{ secrets.MAIL_ADDRESS }}" -e saya_today_web_mail_username="${{ secrets.MAIL_USERNAME }}" -e saya_today_web_mail_password="${{ secrets.MAIL_PASSWORD }}" -e saya_today_google_site_verification="${{ secrets.GOOGLE_SITE_VERIFICATION }}"
+ ansible-playbook -i inventory.yml -l prod playbook.yml --private-key ~/.ssh/id_ed25519 -u svc_github -e saya_today_web_auth_salt="${{ secrets.AUTH_SALT }}" -e saya_today_web_s3_access_key_id="${{ secrets.S3_ACCESS_KEY_ID }}" -e saya_today_web_s3_secret_access_key="${{ secrets.S3_SECRET_ACCESS_KEY }}" -e saya_today_web_environment=prod -e saya_today_web_tag=${{ github.ref_name }} -e saya_today_web_mail_salt="${{ secrets.MAIL_SALT }}" -e saya_today_web_mail_host="${{ secrets.MAIL_HOST }}" -e saya_today_web_mail_address="${{ secrets.MAIL_ADDRESS }}" -e saya_today_web_mail_username="${{ secrets.MAIL_USERNAME }}" -e saya_today_web_mail_password="${{ secrets.MAIL_PASSWORD }}" -e saya_today_google_site_verification="${{ secrets.GOOGLE_SITE_VERIFICATION }}" -e saya_today_yandex_verification="${{ secrets.YANDEX_VERIFICATION }}"
diff --git a/.github/workflows/build-and-deploy-stage.yml b/.github/workflows/build-and-deploy-stage.yml
index e699856..ffee091 100644
--- a/.github/workflows/build-and-deploy-stage.yml
+++ b/.github/workflows/build-and-deploy-stage.yml
@@ -86,4 +86,4 @@ jobs:
ANSIBLE_HOST_KEY_CHECKING: False
working-directory: ./deploy
run: |
- ansible-playbook -i inventory.yml -l stage playbook.yml --private-key ~/.ssh/id_ed25519 -u svc_github -e saya_today_web_auth_salt="${{ secrets.AUTH_SALT }}" -e saya_today_web_b2_key_id="${{ secrets.B2_KEY_ID }}" -e saya_today_web_b2_application_key="${{ secrets.B2_APPLICATION_KEY }}" -e saya_today_web_environment=stage -e saya_today_web_tag=commit-${{ steps.ghss_vars.outputs.sha_short }} -e saya_today_web_mail_salt="${{ secrets.MAIL_SALT }}" -e saya_today_web_mail_host="${{ secrets.MAIL_HOST }}" -e saya_today_web_mail_address="${{ secrets.MAIL_ADDRESS }}" -e saya_today_web_mail_username="${{ secrets.MAIL_USERNAME }}" -e saya_today_web_mail_password="${{ secrets.MAIL_PASSWORD }}"
+ ansible-playbook -i inventory.yml -l stage playbook.yml --private-key ~/.ssh/id_ed25519 -u svc_github -e saya_today_web_auth_salt="${{ secrets.AUTH_SALT }}" -e saya_today_web_s3_access_key_id="${{ secrets.S3_ACCESS_KEY_ID }}" -e saya_today_web_s3_secret_access_key="${{ secrets.S3_SECRET_ACCESS_KEY }}" -e saya_today_web_environment=stage -e saya_today_web_tag=commit-${{ steps.ghss_vars.outputs.sha_short }} -e saya_today_web_mail_salt="${{ secrets.MAIL_SALT }}" -e saya_today_web_mail_host="${{ secrets.MAIL_HOST }}" -e saya_today_web_mail_address="${{ secrets.MAIL_ADDRESS }}" -e saya_today_web_mail_username="${{ secrets.MAIL_USERNAME }}" -e saya_today_web_mail_password="${{ secrets.MAIL_PASSWORD }}"
diff --git a/config/config.go b/config/config.go
index 536a333..0b1570a 100644
--- a/config/config.go
+++ b/config/config.go
@@ -12,6 +12,7 @@ import (
type Config struct {
LogLevel slog.Level `json:"LogLevel" yaml:"logLevel" validate:"required"`
+ Endpoint EndpointConfig `json:"Endpoint" yaml:"endpoint" validate:"required"`
BlogPages BlogPagesConfig `json:"BlogPages" yaml:"blogPages" validate:"required"`
FactGiver FactGiverConfig `json:"FactGiver" yaml:"factGiver" validate:"required"`
LocalePath string `json:"LocalePath" yaml:"localePath" validate:"required,filepath"`
@@ -25,12 +26,90 @@ type Config struct {
AllowOrigins []string `json:"AllowOrigins" yaml:"allowOrigins"`
}
+type EndpointConfig struct {
+ Type string `json:"Type" yaml:"type" validate:"required,oneof=http unix"`
+ Config any `json:"Config" yaml:"config" validate:"required"`
+}
+
+func (ec *EndpointConfig) UnmarshalJSON(data []byte) error {
+ var tmp struct {
+ Type string `json:"Type"`
+ Config json.RawMessage `json:"Config"`
+ }
+
+ if err := json.Unmarshal(data, &tmp); err != nil {
+ return err
+ }
+
+ ec.Type = tmp.Type
+
+ switch tmp.Type {
+ case "http":
+ var httpConfig HttpConfig
+ if err := json.Unmarshal(tmp.Config, &httpConfig); err != nil {
+ return fmt.Errorf("unmarshal HttpConfig: %w", err)
+ }
+ ec.Config = &httpConfig
+ case "unix":
+ var unixConfig UnixConfig
+ if err := json.Unmarshal(tmp.Config, &unixConfig); err != nil {
+ return fmt.Errorf("unmarshal UnixConfig: %w", err)
+ }
+ ec.Config = &unixConfig
+ default:
+ return fmt.Errorf("unsupported storage type: %s", tmp.Type)
+ }
+
+ return nil
+}
+
+func (ec *EndpointConfig) UnmarshalYAML(value *yaml.Node) error {
+ var tmp struct {
+ Type string `yaml:"type"`
+ Config yaml.Node `yaml:"config"`
+ }
+
+ if err := value.Decode(&tmp); err != nil {
+ return err
+ }
+
+ ec.Type = tmp.Type
+
+ switch tmp.Type {
+ case "http":
+ var httpConfig HttpConfig
+ if err := tmp.Config.Decode(&httpConfig); err != nil {
+ return fmt.Errorf("unmarshal HttpConfig: %w", err)
+ }
+ ec.Config = &httpConfig
+ case "unix":
+ var unixConfig UnixConfig
+ if err := tmp.Config.Decode(&unixConfig); err != nil {
+ return fmt.Errorf("unmarshal UnixConfig: %w", err)
+ }
+ ec.Config = &unixConfig
+ default:
+ return fmt.Errorf("unsupported storage type: %s", tmp.Type)
+ }
+
+ return nil
+}
+
+type HttpConfig struct {
+ ListenOn string `json:"ListenOn" yaml:"listenOn" validate:"required"`
+}
+
+type UnixConfig struct {
+ Path string `json:"Path" yaml:"path" validate:"required"`
+ Chmod string `json:"Chmod" yaml:"chmod" validate:"oneof=0600 0660 0666"`
+}
+
type BlogPagesConfig struct {
Storage StorageConfig `json:"Storage" yaml:"storage" validate:"required"`
}
type StorageConfig struct {
- Type string `json:"type" yaml:"type,oneof=b2 s3"`
+ Type string `json:"Type" yaml:"type" validate:"required,oneof=b2 s3"`
Config any `json:"Config" yaml:"config" validate:"required"`
}
@@ -159,6 +238,7 @@ type TriggerConfig struct {
type MetaConfig struct {
GoogleSiteVerification string `json:"GoogleSiteVerification" yaml:"googleSiteVerification"`
+ YandexVerification string `json:"YandexVerification" yaml:"yandexVerification"`
}
type PhotoStorageConfig struct {
@@ -202,6 +282,10 @@ func InitConfig(path string) (*Config, error) {
return nil, err
}
+ if config.Endpoint.Type == "unix" && config.Endpoint.Config.(*UnixConfig).Chmod == "" {
+ config.Endpoint.Config.(*UnixConfig).Chmod = "0660"
+ }
+
validate := validator.New(validator.WithRequiredStructEnabled())
if err := validate.Struct(config); err != nil {
return nil, err
diff --git a/config/config.local.yaml b/config/config.local.yaml
index 7769438..d3e2e33 100644
--- a/config/config.local.yaml
+++ b/config/config.local.yaml
@@ -1,22 +1,30 @@
logLevel: debug
+endpoint:
+ type: http
+ config:
+ listenOn: ":3000"
blogPages:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: "stage-"
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factGiver:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: ""
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factsFileName: "facts-*.txt"
localePath: ./locale/
availableLanguages:
diff --git a/config/config.prod.yaml b/config/config.prod.yaml
index 91f606f..e5785f8 100644
--- a/config/config.prod.yaml
+++ b/config/config.prod.yaml
@@ -1,22 +1,31 @@
logLevel: info
+endpoint:
+ type: unix
+ config:
+ path: "/etc/sayauz/prod.socket"
+ chmod: '0666'
blogPages:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: "prod-"
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factGiver:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: ""
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factsFileName: "facts-*.txt"
localePath: ./locale/
availableLanguages:
@@ -47,6 +56,7 @@ mail:
canonicalEndpoint: "https://${FQDN}"
meta:
googleSiteVerification: "${GOOGLE_SITE_VERIFICATION}"
+ yandexVerification: "${YANDEX_VERIFICATION}"
photoStorage:
full:
baseUrl: https://storage.yandexcloud.kz/sayauz-photos/%s
diff --git a/config/config.stage.yaml b/config/config.stage.yaml
index 3f1381b..c2e337b 100644
--- a/config/config.stage.yaml
+++ b/config/config.stage.yaml
@@ -1,22 +1,31 @@
logLevel: debug
+endpoint:
+ type: unix
+ config:
+ path: "/etc/sayauz/stage.socket"
+ chmod: '0666'
blogPages:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: "stage-"
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factGiver:
storage:
- type: b2
+ type: s3
config:
- bucketName: sayana-pages
- region: eu-central-003
+ bucketName: sayauz-pages
+ region: kz1
prefix: ""
- keyID: "${B2_KEY_ID}"
- applicationKey: "${B2_APPLICATION_KEY}"
+ endpoint: "https://storage.yandexcloud.kz"
+ usePathStyle: true
+ accessKeyID: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
factsFileName: "facts-*.txt"
localePath: ./locale/
availableLanguages:
diff --git a/deploy/inventory.yml b/deploy/inventory.yml
index d3acab8..0a63140 100644
--- a/deploy/inventory.yml
+++ b/deploy/inventory.yml
@@ -8,6 +8,7 @@ all:
saya_today_web_hostname: sayauz-prod
saya_today_web_ipv4_address: 172.16.0.17
saya_today_web_listen_address: saya.uz
+ saya_today_web_unix_sockets_path: '/etc/sayauz'
stage:
hosts:
@@ -17,3 +18,4 @@ all:
saya_today_web_hostname: sayauz-stage
saya_today_web_ipv4_address: 172.16.0.18
saya_today_web_listen_address: stage.saya.uz
+ saya_today_web_unix_sockets_path: '/etc/sayauz'
diff --git a/deploy/playbook.yml b/deploy/playbook.yml
index 8fb9088..ae3e936 100644
--- a/deploy/playbook.yml
+++ b/deploy/playbook.yml
@@ -2,9 +2,27 @@
- name: Deploy saya-today-web Docker Container
hosts: all
remote_user: svc_github
+
+ vars:
+ docker_volumes:
+ - "{{ saya_today_web_name }}-volume:/data:rw"
tasks:
- - name: Create a volume
+ - name: Initialize Unix socket volume
+ when: >-
+ saya_today_web_unix_sockets_path is defined and
+ saya_today_web_unix_sockets_path is string and
+ saya_today_web_unix_sockets_path | length > 0
+ block:
+ - name: Ensure Unix socket volume exists
+ community.docker.docker_volume:
+ name: "sayana_sockets"
+ - name: Include Unix socket volume in docker volumes list
+ ansible.builtin.set_fact:
+ docker_volumes: >-
+ {{ docker_volumes + ["sayana_sockets:" ~ saya_today_web_unix_sockets_path ~ ":rw"] }}
+
+ - name: Create a data volume
community.docker.docker_volume:
name: "{{ saya_today_web_name }}-volume"
@@ -14,8 +32,8 @@
hostname: "{{ saya_today_web_hostname }}"
image: "ghcr.io/sayaandy/saya-today-web:{{ saya_today_web_tag }}"
env:
- B2_KEY_ID: "{{ saya_today_web_b2_key_id }}"
- B2_APPLICATION_KEY: "{{ saya_today_web_b2_application_key }}"
+ S3_ACCESS_KEY_ID: "{{ saya_today_web_s3_access_key_id }}"
+ S3_SECRET_ACCESS_KEY: "{{ saya_today_web_s3_secret_access_key }}"
ENVIRONMENT: "{{ saya_today_web_environment }}"
AUTH_SALT: "{{ saya_today_web_auth_salt }}"
MAIL_HOST: "{{ saya_today_web_mail_host }}"
@@ -25,12 +43,12 @@
MAIL_SALT: "{{ saya_today_web_mail_salt }}"
FQDN: "{{ saya_today_web_listen_address }}"
GOOGLE_SITE_VERIFICATION: "{{ saya_today_google_site_verification | default('') }}"
+ YANDEX_VERIFICATION: "{{ saya_today_yandex_verification | default('') }}"
network_mode: caddy
networks:
- name: caddy
ipv4_address: "{{ saya_today_web_ipv4_address }}"
- volumes:
- - "{{ saya_today_web_name }}-volume:/data:rw"
+ volumes: "{{ docker_volumes }}"
restart_policy: unless-stopped
no_log: true
...
diff --git a/go.mod b/go.mod
index 7d44551..779e641 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,6 @@ require (
require (
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
- github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
diff --git a/go.sum b/go.sum
index 597f315..d277091 100644
--- a/go.sum
+++ b/go.sum
@@ -2,78 +2,40 @@ github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQ
github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
-github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
-github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
-github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
-github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c=
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
-github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
-github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -91,8 +53,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
-github.com/go-co-op/gocron/v2 v2.20.0 h1:9IMrnnVSWjfSh3E54gWmWCHbloQJLh6f9+nwyKfLNpc=
-github.com/go-co-op/gocron/v2 v2.20.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
github.com/go-co-op/gocron/v2 v2.21.1 h1:QYOK6iOQVCut+jDcs4zRdWRTBHRxRCEeeFi1TnAmgbU=
github.com/go-co-op/gocron/v2 v2.21.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@@ -103,8 +63,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK2xqPNk8vgvu5JQ=
github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc=
-github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw=
-github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk=
github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
@@ -113,8 +71,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
-github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
-github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -127,20 +83,12 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
-github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
-github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
-github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA=
-github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8=
github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -152,8 +100,6 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po=
-github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -168,17 +114,10 @@ github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
-golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/internal/blog/index.go b/internal/blog/index.go
new file mode 100644
index 0000000..861a90e
--- /dev/null
+++ b/internal/blog/index.go
@@ -0,0 +1,32 @@
+package blog
+
+import "time"
+
+const IndexFileName = "index.json"
+
+const IndexSchemaVersion = 1
+
+type IndexEntry struct {
+ Link string `json:"link"`
+ ModifiedTime time.Time `json:"modifiedTime"`
+ Title string `json:"title"`
+ ShortDescription string `json:"shortDescription"`
+ ActionDate string `json:"actionDate"`
+ PublishedTime time.Time `json:"publishedTime"`
+ Thumbnail string `json:"thumbnail"`
+ Tags []string `json:"tags"`
+ Geolocation string `json:"geolocation"`
+ Medley string `json:"medley,omitempty"`
+ MedleyPart int `json:"medleyPart,omitempty"`
+}
+
+type IndexCategory struct {
+ GeneratedAt time.Time `json:"generatedAt"`
+ Pages []IndexEntry `json:"pages"`
+}
+
+type Index struct {
+ SchemaVersion int `json:"schemaVersion"`
+ GeneratedAt time.Time `json:"generatedAt"`
+ Categories map[string]IndexCategory `json:"categories"`
+}
diff --git a/internal/blog/s3.go b/internal/blog/s3.go
index 86a6463..bbd5238 100644
--- a/internal/blog/s3.go
+++ b/internal/blog/s3.go
@@ -2,10 +2,15 @@ package blog
import (
"context"
+ "encoding/json"
+ "errors"
"fmt"
"io"
+ "log/slog"
+ "net/url"
"slices"
"strings"
+ "sync"
"time"
"github.com/SayaAndy/saya-today-web/config"
@@ -14,8 +19,11 @@ import (
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
+ s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
+const s3ScanConcurrency = 32
+
type S3Client struct {
prefix string
bucketName string
@@ -50,6 +58,7 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) {
}
s3Opts = append(s3Opts, func(o *s3.Options) {
o.UsePathStyle = s3cfg.UsePathStyle
+ o.DisableLogOutputChecksumValidationSkipped = true
})
s3cl := s3.NewFromConfig(awsCfg, s3Opts...)
@@ -58,79 +67,195 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) {
}
func (c *S3Client) Scan(prefix string) ([]*Page, error) {
- pages := []*Page{}
+ pages, err := c.scanFromIndex(prefix)
+ if err == nil {
+ return pages, nil
+ }
- fullPrefix := c.prefix + prefix
- input := &s3.ListObjectsV2Input{
+ var nsk *s3types.NoSuchKey
+ if !errors.As(err, &nsk) {
+ return nil, err
+ }
+
+ slog.Warn("index.json missing, falling back to listing", slog.String("prefix", c.prefix))
+ return c.scanByListing(prefix)
+}
+
+func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) {
+ out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: aws.String(c.bucketName),
- Prefix: aws.String(fullPrefix),
+ Key: aws.String(IndexFileName),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("get index.json: %w", err)
+ }
+ defer out.Body.Close()
+
+ raw, err := io.ReadAll(out.Body)
+ if err != nil {
+ return nil, fmt.Errorf("read index.json: %w", err)
+ }
+
+ var idx Index
+ if err := json.Unmarshal(raw, &idx); err != nil {
+ return nil, fmt.Errorf("unmarshal index.json: %w", err)
+ }
+
+ wantLang := ""
+ if i := strings.Index(prefix, "/"); i > 0 {
+ wantLang = prefix[:i]
+ }
+
+ fullPrefix := c.prefix + prefix
+ pages := make([]*Page, 0)
+ for catKey, cat := range idx.Categories {
+ lang, ok := strings.CutPrefix(catKey, c.prefix)
+ if !ok {
+ continue
+ }
+ if wantLang != "" && wantLang != lang {
+ continue
+ }
+ for _, e := range cat.Pages {
+ if !strings.HasPrefix(e.Link, fullPrefix) {
+ continue
+ }
+ linkParts := strings.Split(e.Link, "/")
+ nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
+ fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
+ pages = append(pages, &Page{
+ Link: e.Link,
+ FileName: fileName,
+ Lang: lang,
+ ModifiedTime: e.ModifiedTime,
+ Metadata: &frontmatter.Metadata{
+ Title: e.Title,
+ ShortDescription: e.ShortDescription,
+ ActionDate: e.ActionDate,
+ PublishedTime: e.PublishedTime,
+ Thumbnail: e.Thumbnail,
+ Tags: e.Tags,
+ Geolocation: e.Geolocation,
+ Medley: e.Medley,
+ MedleyPart: e.MedleyPart,
+ },
+ })
+ }
}
+ return pages, nil
+}
- paginator := s3.NewListObjectsV2Paginator(c.s3cl, input)
+func (c *S3Client) scanByListing(prefix string) ([]*Page, error) {
+ fullPrefix := c.prefix + prefix
+
+ type candidate struct {
+ key string
+ lastModified time.Time
+ }
+ var candidates []candidate
+ paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(fullPrefix),
+ })
for paginator.HasMorePages() {
output, err := paginator.NextPage(context.Background())
if err != nil {
return nil, fmt.Errorf("list S3 objects: %w", err)
}
-
for _, obj := range output.Contents {
key := aws.ToString(obj.Key)
-
if !strings.HasSuffix(key, ".md") {
continue
}
+ candidates = append(candidates, candidate{key, aws.ToTime(obj.LastModified)})
+ }
+ }
+
+ pages := make([]*Page, len(candidates))
+ sem := make(chan struct{}, s3ScanConcurrency)
+ var wg sync.WaitGroup
+ var firstErr error
+ var errMu sync.Mutex
+
+ for i, cand := range candidates {
+ sem <- struct{}{}
+ wg.Go(func() {
+ defer func() { <-sem }()
head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
+ Key: aws.String(cand.key),
})
if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
+ errMu.Lock()
+ if firstErr == nil {
+ firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err)
+ }
+ errMu.Unlock()
+ return
}
if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
- continue
+ return
}
-
meta := head.Metadata
if meta["title"] == "" {
- continue
+ return
}
publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
if err != nil {
- return nil, fmt.Errorf("failed to parse published time metadata field: %w", err)
+ errMu.Lock()
+ if firstErr == nil {
+ firstErr = fmt.Errorf("failed to parse published time metadata field: %w", err)
+ }
+ errMu.Unlock()
+ return
}
- linkParts := strings.Split(key, "/")
+ linkParts := strings.Split(cand.key, "/")
nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
+ lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
tags := strings.Split(meta["tags"], ",")
slices.Sort(tags)
- lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
+ title, _ := url.QueryUnescape(meta["title"])
+ shortDescription, _ := url.QueryUnescape(meta["short-description"])
+ thumbnail, _ := url.QueryUnescape(meta["thumbnail"])
- pages = append(pages, &Page{
- Link: key,
+ pages[i] = &Page{
+ Link: cand.key,
FileName: fileName,
Lang: lang,
- ModifiedTime: aws.ToTime(obj.LastModified),
+ ModifiedTime: cand.lastModified,
Metadata: &frontmatter.Metadata{
- Title: meta["title"],
- ShortDescription: meta["short-description"],
+ Title: title,
+ ShortDescription: shortDescription,
ActionDate: meta["action-date"],
PublishedTime: publishedTime,
- Thumbnail: meta["thumbnail"],
+ Thumbnail: thumbnail,
Tags: tags,
Geolocation: meta["geolocation"],
},
- })
- }
+ }
+ })
}
+ wg.Wait()
- return pages, nil
+ if firstErr != nil {
+ return nil, firstErr
+ }
+
+ out := pages[:0]
+ for _, p := range pages {
+ if p != nil {
+ out = append(out, p)
+ }
+ }
+ return out, nil
}
func (c *S3Client) ReadAll(path string) ([]byte, error) {
diff --git a/internal/router/handlers/root.go b/internal/router/handlers/root.go
index a8890fa..6a011e4 100644
--- a/internal/router/handlers/root.go
+++ b/internal/router/handlers/root.go
@@ -50,6 +50,9 @@ func (r *RootHandler) AddMeta(c *fiber.Ctx, supplements *router.Supplements, lan
if supplements.Meta.GoogleSiteVerification != "" {
meta = append(meta, router.MetaField{Name: "google-site-verification", Content: supplements.Meta.GoogleSiteVerification})
}
+ if supplements.Meta.YandexVerification != "" {
+ meta = append(meta, router.MetaField{Name: "yandex-verification", Content: supplements.Meta.YandexVerification})
+ }
return meta, nil
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 5230e74..65e35a4 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -7,8 +7,11 @@ import (
"fmt"
"html/template"
"log/slog"
+ "net"
"net/url"
+ "os"
"slices"
+ "strconv"
"strings"
"time"
@@ -112,6 +115,7 @@ type Router struct {
templatedRoutes map[string]map[string]Route
templatedPathMatcher *PathMatcher
canonicalEndpoint string
+ endpoint config.EndpointConfig
}
func NewRouter(cfg *config.Config) (*Router, error) {
@@ -256,7 +260,7 @@ func NewRouter(cfg *config.Config) (*Router, error) {
templatedRoutes := make(map[string]map[string]Route)
templatedPathMatcher := NewPathMatcher()
- return &Router{supplements, app, templatedRoutes, templatedPathMatcher, cfg.CanonicalEndpoint}, nil
+ return &Router{supplements, app, templatedRoutes, templatedPathMatcher, cfg.CanonicalEndpoint, cfg.Endpoint}, nil
}
func (r *Router) InitRoutes() (err error) {
@@ -305,6 +309,8 @@ func (r *Router) InitRoutes() (err error) {
cacheKey = fmt.Sprintf("%s.full-page.%s", method, trimmedPath)
case ByUrlAndQuery:
cacheKey = fmt.Sprintf("%s.full-page.%s.%s", method, trimmedPath, queryString)
+ case Disabled:
+ c.Set("Cache-Control", "no-store, no-cache, must-revalidate")
}
defaultMap := fiber.Map{
@@ -367,6 +373,7 @@ func (r *Router) InitRoutes() (err error) {
c.Set(fiber.HeaderContentType, fiber.MIMETextPlainCharsetUTF8)
return c.Status(fiber.ErrBadRequest.Code).SendString(fmt.Sprintf("unknown segment '%s'", part))
}
+ c.Set("Cache-Control", "no-store, no-cache, must-revalidate")
err := r.generalPageSegment(c, part)
return err
})
@@ -382,10 +389,30 @@ func (r *Router) InitRoutes() (err error) {
return nil
}
-func (r *Router) Listen(endpoint string) error {
- if err := r.app.Listen(endpoint); err != nil {
- return fmt.Errorf("error while running fiber server: %w", err)
+func (r *Router) Listen() error {
+ switch r.endpoint.Type {
+ case "unix":
+ unixConfig := r.endpoint.Config.(*config.UnixConfig)
+ endpoint, _ := strings.CutPrefix(unixConfig.Path, "unix://")
+ ln, err := net.Listen("unix", endpoint)
+ if err != nil {
+ return fmt.Errorf("error while initializing unix listener: %w", err)
+ }
+ chmod, _ := strconv.ParseUint(unixConfig.Chmod[1:], 8, 32)
+ os.Chmod(unixConfig.Path, os.FileMode(chmod))
+ if err := r.app.Listener(ln); err != nil {
+ return fmt.Errorf("error while running fiber server: %w", err)
+ }
+ case "http":
+ httpConfig := r.endpoint.Config.(*config.HttpConfig)
+ fmt.Print(httpConfig)
+ if err := r.app.Listen(httpConfig.ListenOn); err != nil {
+ return fmt.Errorf("error while running fiber server: %w", err)
+ }
+ default:
+ return fmt.Errorf("error with initializing fiber server: invalid endpoint type (supported are unix and http)")
}
+
return nil
}
@@ -409,6 +436,7 @@ func (r *Router) Close() (err error) {
}
slog.Debug("closing page cache")
r.supplements.PageCache.Close()
+
return errors.Join(allErrors...)
}
diff --git a/main.go b/main.go
index 3691e5b..9d90916 100644
--- a/main.go
+++ b/main.go
@@ -47,7 +47,7 @@ func main() {
routerShutdown := make(chan struct{}, 1)
go func() {
- if err := app.Listen(":3000"); err != nil {
+ if err := app.Listen(); err != nil {
slog.Error("error while running fiber server", slog.String("error", err.Error()))
if nestedErr := app.Close(); nestedErr != nil {
slog.Error("error while shutting down fiber server", slog.String("error", err.Error()))
diff --git a/static/input.css b/static/input.css
index 75482d8..cdab1f0 100644
--- a/static/input.css
+++ b/static/input.css
@@ -1,5 +1,3 @@
-@import url('https://fonts.googleapis.com/css2?family=Gentium+Plus:ital,wght@0,400;0,700;1,400;1,700&family=M+PLUS+Rounded+1c:wght@100;300;400;500;700;800;900&display=swap');
-
@import "tailwindcss";
@plugin "@tailwindcss/forms";
diff --git a/views/layouts/general-page.html b/views/layouts/general-page.html
index 93567d3..2f949a3 100644
--- a/views/layouts/general-page.html
+++ b/views/layouts/general-page.html
@@ -12,12 +12,13 @@
<link rel="canonical" href="{{.Content}}">
{{- end }}
{{- end }}
+ <link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/fonts/fonts.css">
<link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/libs/font-awesome/7.2.0/css/fontawesome.min.css">
<link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/libs/font-awesome/7.2.0/css/brands.min.css">
<link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/libs/font-awesome/7.2.0/css/solid.min.css">
<link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/libs/leaflet/1.9.4/leaflet.css">
<link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/libs/glightbox/3.3.0/css/glightbox.min.css">
- <link href="/output.css" rel="stylesheet">
+ <link rel="stylesheet" href="/output.css">
{{- with .LinkedData }}
<script type="application/ld+json">
{{ . }}
diff --git a/views/pages/global-map.html b/views/pages/global-map.html
index aa6ddfd..bb51131 100644
--- a/views/pages/global-map.html
+++ b/views/pages/global-map.html
@@ -15,7 +15,8 @@
/>
<meta property="og:type" content="site" />
<title>{{ .L.GlobalMap.Header }} // SAYA.UZ</title>
- <link rel="canonical" href="{{ .CanonicalEndpoint }}/{{ .Lang }}/map" />
+ <link rel="canonical" href="{{ .CanonicalEndpoint }}/{{ .Lang }}/map">
+ <link rel="stylesheet" href="{{ .StaticStorageBaseUrl }}/fonts/fonts.css">
<link
rel="stylesheet"
href="{{ .StaticStorageBaseUrl }}/libs/font-awesome/7.2.0/css/fontawesome.min.css"
@@ -29,7 +30,7 @@
href="{{ .StaticStorageBaseUrl }}/libs/font-awesome/7.2.0/css/solid.min.css"
/>
{{ block "header" . }}{{ end }}
- <link href="/output.css" rel="stylesheet" />
+ <link rel="stylesheet" href="/output.css">
<link
rel="stylesheet"
href="{{ .StaticStorageBaseUrl }}/libs/leaflet/1.9.4/leaflet.css"
diff --git a/views/pages/robots.txt b/views/pages/robots.txt
index 1e5bebb..aaef3f4 100644
--- a/views/pages/robots.txt
+++ b/views/pages/robots.txt
@@ -1,5 +1,7 @@
User-agent: *
Allow: /
-Disallow: /api/
+Disallow: /api/v1/email/
+Disallow: /ru/user/
+Disallow: /en/user/
Sitemap: {{ .CanonicalEndpoint }}/sitemap.xml \ No newline at end of file
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-08 20:07:58 +0000