royalcat/tstor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages1 Wiki Activity Actions

rework

Browse source
This commit is contained in:
royalcat 2025-04-03 03:19:01 +04:00
parent e62df2e9fc
commit 1912b5baf0
39 changed files with 8489 additions and 476 deletions

4
server/.gqlgen.yml
View file

@ -1,6 +1,6 @@
schema:
- ../graphql/*.graphql
- ../graphql/**/*.graphql
- ./graphql/*.graphql
- ./graphql/**/*.graphql
exec:
filename: src/delivery/graphql/generated.go

30
server/go.mod
View file

@ -9,9 +9,7 @@ toolchain go1.24.1
// replace github.com/chenzhuoyu/iasm v0.9.0 => github.com/cloudwego/iasm v0.2.0
require (
github.com/99designs/gqlgen v0.17.68
github.com/agoda-com/opentelemetry-go/otelslog v0.2.0
github.com/agoda-com/opentelemetry-logs-go v0.5.1
github.com/99designs/gqlgen v0.17.70
github.com/billziss-gh/cgofuse v1.5.0
github.com/cespare/xxhash/v2 v2.3.0
github.com/dgraph-io/badger/v4 v4.5.0
@ -47,20 +45,25 @@ require (
github.com/vektah/gqlparser/v2 v2.5.23
github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00
github.com/willscott/memphis v0.0.0-20210922141505-529d4987ab7e
go.opentelemetry.io/contrib/bridges/otelslog v0.10.0
go.opentelemetry.io/otel v1.35.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0
go.opentelemetry.io/otel/exporters/prometheus v0.53.0
go.opentelemetry.io/otel/metric v1.35.0
go.opentelemetry.io/otel/sdk v1.31.0
go.opentelemetry.io/otel/sdk/metric v1.31.0
go.opentelemetry.io/otel/sdk v1.35.0
go.opentelemetry.io/otel/sdk/log v0.11.0
go.opentelemetry.io/otel/sdk/metric v1.34.0
go.opentelemetry.io/otel/trace v1.35.0
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
golang.org/x/net v0.37.0
golang.org/x/net v0.38.0
golang.org/x/sync v0.12.0
golang.org/x/sys v0.31.0
)
replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20250324211829-b45e905df463
require (
github.com/agnivade/levenshtein v1.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@ -81,7 +84,7 @@ require (
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/labstack/gommon v0.4.2 // indirect
@ -109,16 +112,17 @@ require (
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib v1.26.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.opentelemetry.io/otel/log v0.11.0 // indirect
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/time v0.10.0 // indirect
golang.org/x/tools v0.31.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.36.5 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
google.golang.org/grpc v1.71.1 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1693
server/go.sum
View file

File diff suppressed because it is too large Load diff

8
server/graphql/mutation.graphql Normal file
View file

@ -0,0 +1,8 @@
type Mutation {
uploadFile(dir: String!, file: Upload!): Boolean!
dedupeStorage: Int!
}
type Task {
id: ID!
}

9
server/graphql/query.graphql Normal file
View file

@ -0,0 +1,9 @@
type Query {
plugins: [Plugin!]!
fsEntry(path: String!): FsEntry
}
type Plugin {
name: String!
endpountSubPath: String
}

13
server/graphql/schema.graphql Normal file
View file

@ -0,0 +1,13 @@
# directive @oneOf on INPUT_OBJECT | FIELD_DEFINITION
directive @resolver on INPUT_FIELD_DEFINITION | FIELD_DEFINITION
directive @stream on FIELD_DEFINITION
scalar DateTime
scalar Upload
scalar UInt
type Schema {
query: Query
mutation: Mutation
}

3
server/graphql/subscription.graphql Normal file
View file

@ -0,0 +1,3 @@
type Subscription {
taskProgress(taskID: ID!): Progress
}

31
server/graphql/types/filters.graphql Normal file
View file

@ -0,0 +1,31 @@
input Pagination {
offset: Int!
limit: Int!
}
input StringFilter @oneOf {
eq: String
substr: String
in: [String!]
}
input IntFilter @oneOf {
eq: Int
gt: Int
lt: Int
gte: Int
lte: Int
in: [Int!]
}
input DateTimeFilter @oneOf {
eq: DateTime
gt: DateTime
lt: DateTime
gte: DateTime
lte: DateTime
}
input BooleanFilter @oneOf {
eq: Boolean
}

35
server/graphql/types/fs.graphql Normal file
View file

@ -0,0 +1,35 @@
interface FsEntry {
name: String!
}
interface Dir implements FsEntry {
name: String!
entries: [FsEntry!]!
}
interface File implements FsEntry {
name: String!
size: Int!
}
type SimpleDir implements Dir & FsEntry {
name: String!
entries: [FsEntry!]! @resolver
}
type SimpleFile implements File & FsEntry {
name: String!
size: Int!
}
type ResolverFS implements Dir & FsEntry {
name: String!
entries: [FsEntry!]! @resolver
}
# type ArchiveFS implements Dir & FsEntry {
# name: String!
# entries: [FsEntry!]! @resolver
# size: Int!
# }

4
server/graphql/types/progress.graphql Normal file
View file

@ -0,0 +1,4 @@
interface Progress {
current: Int!
total: Int!
}

35
server/src/telemetry/setup.go
View file

@ -3,22 +3,20 @@ package telemetry
import (
"context"
"fmt"
"log/slog"
"os"
"runtime"
"git.kmsign.ru/royalcat/tstor/server/pkg/rlog"
"github.com/agoda-com/opentelemetry-go/otelslog"
"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs"
"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogshttp"
logsdk "github.com/agoda-com/opentelemetry-logs-go/sdk/logs"
"github.com/google/uuid"
otelpyroscope "github.com/grafana/otel-profiling-go"
"github.com/grafana/pyroscope-go"
"go.opentelemetry.io/contrib/bridges/otelslog"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
@ -30,7 +28,7 @@ type Client struct {
tracerProvider *trace.TracerProvider
metricProvider *metric.MeterProvider
loggerProvider *logsdk.LoggerProvider
loggerProvider *log.LoggerProvider
}
func (client *Client) Shutdown(ctx context.Context) {
@ -124,25 +122,22 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) {
otel.SetTracerProvider(otelpyroscope.NewTracerProvider(client.tracerProvider))
client.log.Info(ctx, "tracing provider initialized")
logExporter, err := otlplogs.NewExporter(ctx,
otlplogs.WithClient(
otlplogshttp.NewClient(
otlplogshttp.WithEndpoint(endpoint),
),
),
logExporter, err := otlploghttp.New(ctx,
otlploghttp.WithEndpoint(endpoint),
otlploghttp.WithRetry(otlploghttp.RetryConfig{
Enabled: false,
}),
)
if err != nil {
return nil, err
}
client.loggerProvider = logsdk.NewLoggerProvider(
logsdk.WithBatcher(logExporter),
logsdk.WithResource(r),
)
rlog.AddHandler(otelslog.NewOtelHandler(client.loggerProvider,
&otelslog.HandlerOptions{
Level: slog.LevelDebug,
}),
client.loggerProvider = log.NewLoggerProvider(
log.WithProcessor(log.NewBatchProcessor(logExporter)),
)
rlog.AddHandler(otelslog.NewHandler("", otelslog.WithLoggerProvider(client.loggerProvider)))
client.log.Info(ctx, "logger provider initialized")
// recreate telemetry logger

17
server/tstor/plugin.go
View file

@ -1,9 +1,18 @@
package tstor
import "git.kmsign.ru/royalcat/tstor/server/src/daemon"
import (
"context"
type Plugin struct {
Name string
"git.kmsign.ru/royalcat/tstor/server/src/daemon"
"github.com/99designs/gqlgen/graphql"
"github.com/knadh/koanf/v2"
)
DaemonConstructor daemon.DaemonConstructor
type Plugin interface {
Name() string
}
type DaemonPlugin interface {
NewDaemon(context.Context, *koanf.Koanf) (daemon.Daemon, error)
GraphQLExecutableSchema() graphql.ExecutableSchema
}

26
server/tstor/run.go
View file

@ -36,7 +36,7 @@ const (
webDAVPortFlag = "webdav-port"
)
func Run(plugins []*Plugin) {
func Run(plugins []Plugin) {
app := &cli.App{
Name: "tstor",
Flags: []cli.Flag{
@ -59,7 +59,7 @@ func Run(plugins []*Plugin) {
}
}
func start(configPath string, plugins []*Plugin) error {
func start(configPath string, plugins []Plugin) error {
conf, koanf, err := config.Load(configPath)
if err != nil {
return fmt.Errorf("error loading configuration: %w", err)
@ -88,14 +88,18 @@ func start(configPath string, plugins []*Plugin) error {
pluginConfig := getPluginKoanf(koanf, plugin.Name)
daemon, err := pluginStub.DaemonConstructor(ctx, pluginConfig)
if err != nil {
log.Error(ctx, "error creating plugin daemon", rlog.Error(err))
continue
}
if daemonPlugin, ok := pluginStub.(DaemonPlugin); ok {
log.Debug(ctx, "plugin implements has daemon, creating", slog.String("name", pluginStub.Name()))
log.Debug(ctx, "plugin daemon created", slog.String("name", pluginStub.Name))
daemons = append(daemons, daemon)
daemon, err := daemonPlugin.NewDaemon(ctx, pluginConfig)
if err != nil {
log.Error(ctx, "error creating plugin daemon", rlog.Error(err))
continue
}
log.Debug(ctx, "plugin daemon created", slog.String("name", pluginStub.Name()))
daemons = append(daemons, daemon)
}
}
// TODO make optional
@ -232,9 +236,9 @@ func getPluginKoanf(rootKoanf *koanf.Koanf, name string) *koanf.Koanf {
return nil
}
func getPlugin(plugins []*Plugin, name string) *Plugin {
func getPlugin(plugins []Plugin, name string) Plugin {
for _, p := range plugins {
if p.Name == name {
if p.Name() == name {
return p
}
}