From 06153d61c9460b11f3000bf7f1a1c0df6e034cc9 Mon Sep 17 00:00:00 2001 From: royalcat Date: Mon, 3 Jun 2024 00:20:43 +0300 Subject: [PATCH] fix --- cmd/tstor/main.go | 6 ++- go.mod | 24 +++++------ go.sum | 52 ++++++++++++------------ src/sources/source.go | 53 ------------------------ src/sources/torrent/service.go | 4 +- src/sources/ytdlp/controller.go | 29 +++++++++++-- src/sources/ytdlp/ytdlp.go | 3 +- src/telemetry/setup.go | 72 ++++++++++++++++----------------- 8 files changed, 108 insertions(+), 135 deletions(-) delete mode 100644 src/sources/source.go diff --git a/cmd/tstor/main.go b/cmd/tstor/main.go index c842573..e5041a0 100644 --- a/cmd/tstor/main.go +++ b/cmd/tstor/main.go @@ -96,7 +96,11 @@ func run(configPath string) error { return fmt.Errorf("error creating service: %w", err) } - ytdlpsrv := ytdlp.NewService(conf.SourceDir) + err = os.MkdirAll("./ytdlp", 0744) + if err != nil { + return err + } + ytdlpsrv := ytdlp.NewService("./ytdlp") sfs := sources.NewHostedFS( vfs.NewCtxBillyFs("/", ctxbilly.WrapFileSystem(sourceFs)), diff --git a/go.mod b/go.mod index edabe37..678da9a 100644 --- a/go.mod +++ b/go.mod @@ -45,15 +45,15 @@ require ( github.com/vektah/gqlparser/v2 v2.5.11 github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 github.com/willscott/memphis v0.0.0-20210922141505-529d4987ab7e - go.opentelemetry.io/otel v1.26.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 - go.opentelemetry.io/otel/metric v1.26.0 - go.opentelemetry.io/otel/sdk v1.26.0 - go.opentelemetry.io/otel/sdk/metric v1.26.0 - go.opentelemetry.io/otel/trace v1.26.0 + go.opentelemetry.io/otel/metric v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/sdk/metric v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f - golang.org/x/net v0.24.0 + golang.org/x/net v0.25.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 ) @@ -116,7 +116,7 @@ require ( github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.7 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -183,10 +183,10 @@ require ( golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.20.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.34.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.50.5 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index d271264..843d175 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88= github.com/grafana/pyroscope-go/godeltaprof v0.1.7 h1:C11j63y7gymiW8VugJ9ZW0pWfxTZugdSJyC48olk5KY= github.com/grafana/pyroscope-go/godeltaprof v0.1.7/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -526,8 +526,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/royalcat/ctxio v0.0.0-20240602060200-590d464c39be h1:Ui+Imq1Vk26rfpkLUsgvVdYO/UOJkzDyPzESfrTqWfM= github.com/royalcat/ctxio v0.0.0-20240602060200-590d464c39be/go.mod h1:NFNp3OsEMUPYj5LZUFDiyDt+2E6gR/g8JLd0k+y8XWI= github.com/royalcat/ctxprogress v0.0.0-20240511091748-6d9b327537c3 h1:1Ow/NUAWFZLghFcdNuyHt5Avb+bEI11qG8ELr9/XmQQ= @@ -638,25 +638,25 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib v1.26.0 h1:8/CmxLl5uDm37V9rdqbZcVLvYigAE2vMostBq3nGDrI= go.opentelemetry.io/contrib v1.26.0/go.mod h1:Tmhw9grdWtmXy6DxZNpIAudzYJqLeEM2P6QTZQSRwU8= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 h1:+hm+I+KigBy3M24/h1p/NHkUx/evbLH0PNcjpMyCHc4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0/go.mod h1:NjC8142mLvvNT6biDpaMjyz78kyEHIwAJlSX0N9P5KI= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= -go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZHcAyHw5aU9Y= -go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -748,8 +748,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -905,10 +905,10 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww= -google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -919,8 +919,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -930,8 +930,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/sources/source.go b/src/sources/source.go deleted file mode 100644 index 1629e2c..0000000 --- a/src/sources/source.go +++ /dev/null @@ -1,53 +0,0 @@ -package sources - -import ( - "context" - "encoding/json" - "fmt" - "reflect" -) - -type UpdateTask interface{} - -type Source interface { - Name() string // unique name within source type - SourceType() string - Fetch(ctx context.Context, task UpdateTask, dir string) error -} - -var sourceTypesRegistry = map[string]reflect.Type{} - -// func RegisterSource[T Source]() { -// var s T -// t := reflect.TypeOf(s) -// if t.Kind() == reflect.Ptr { -// RegisterSource[T]() -// return -// } - -// sourceTypesRegistry[s.SourceType()] = t -// } - -type sourceType struct { - Type string `json:"type"` -} - -func parseSource(data []byte) (Source, error) { - var sourceType sourceType - err := json.Unmarshal(data, &sourceType) - if err != nil { - return nil, err - } - - st, ok := sourceTypesRegistry[sourceType.Type] - if !ok { - return nil, fmt.Errorf("source type %s not registred", sourceType.Type) - } - - s := reflect.New(st).Interface().(Source) - err = json.Unmarshal(data, &s) - if err != nil { - return nil, err - } - return s, nil -} diff --git a/src/sources/torrent/service.go b/src/sources/torrent/service.go index d21eace..6ae24dd 100644 --- a/src/sources/torrent/service.go +++ b/src/sources/torrent/service.go @@ -92,11 +92,11 @@ func NewService(sourceFs billy.Filesystem, conf config.TorrentClient) (*Service, return nil, fmt.Errorf("error creating node ID: %w", err) } - client, err := newClient(s.Storage, s.fis, &conf, id) + s.client, err = newClient(s.Storage, s.fis, &conf, id) if err != nil { return nil, fmt.Errorf("error starting torrent client: %w", err) } - client.AddDhtNodes(conf.DHTNodes) + s.client.AddDhtNodes(conf.DHTNodes) s.dirsAquire, err = NewKV[string, DirAquire](conf.MetadataFolder, "dir-acquire") if err != nil { diff --git a/src/sources/ytdlp/controller.go b/src/sources/ytdlp/controller.go index 03233dc..a2c088c 100644 --- a/src/sources/ytdlp/controller.go +++ b/src/sources/ytdlp/controller.go @@ -6,18 +6,39 @@ import ( "fmt" "path" "sync" + "time" "git.kmsign.ru/royalcat/tstor/pkg/ctxbilly" "git.kmsign.ru/royalcat/tstor/src/vfs" "github.com/go-git/go-billy/v5/osfs" "github.com/royalcat/ctxio" + "github.com/royalcat/ctxprogress" ) func NewService(dataDir string) *Service { - return &Service{ + s := &Service{ dataDir: dataDir, sources: make(map[string]ytdlpSource, 0), } + + go func() { + for { + ctx := context.Background() + ctx = ctxprogress.New(ctx) + ctxprogress.AddCallback(ctx, func(p ctxprogress.Progress) { + cur, total := p.Progress() + fmt.Printf("updating sources: %d/%d\n", cur, total) + }) + + err := s.Update(ctx) + if err != nil { + fmt.Println("failed to update sources:", err) + } + time.Sleep(time.Minute) + } + }() + + return s } type Service struct { @@ -27,7 +48,7 @@ type Service struct { sources map[string]ytdlpSource } -func (c *Service) AddSource(s ytdlpSource) { +func (c *Service) addSource(s ytdlpSource) { c.mu.Lock() defer c.mu.Unlock() @@ -65,9 +86,9 @@ func (c *Service) BuildFS(ctx context.Context, f vfs.File) (vfs.Filesystem, erro return nil, err } - c.AddSource(s) + c.addSource(s) downloadFS := ctxbilly.WrapFileSystem(osfs.New(c.sourceDir(s))) - return newSourceFS(s.Name(), downloadFS, c, s), nil + return newSourceFS(path.Base(f.Name()), downloadFS, c, s), nil } diff --git a/src/sources/ytdlp/ytdlp.go b/src/sources/ytdlp/ytdlp.go index 916aa30..1318701 100644 --- a/src/sources/ytdlp/ytdlp.go +++ b/src/sources/ytdlp/ytdlp.go @@ -3,6 +3,7 @@ package ytdlp import ( "context" "crypto/sha1" + "encoding/base64" "git.kmsign.ru/royalcat/tstor/pkg/ytdlp" "github.com/royalcat/ctxprogress" @@ -15,7 +16,7 @@ type ytdlpSource struct { var hasher = sha1.New() func (s *ytdlpSource) Name() string { - return string(hasher.Sum([]byte(s.Url))) + return base64.URLEncoding.EncodeToString(hasher.Sum([]byte(s.Url))) } func (s *ytdlpSource) Download(ctx context.Context, task TaskUpdater, dir string) error { diff --git a/src/telemetry/setup.go b/src/telemetry/setup.go index a952626..5339be8 100644 --- a/src/telemetry/setup.go +++ b/src/telemetry/setup.go @@ -10,17 +10,17 @@ import ( "git.kmsign.ru/royalcat/tstor/pkg/rlog" "github.com/agoda-com/opentelemetry-go/otelslog" "github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs" - "github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogsgrpc" + "github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogshttp" logsdk "github.com/agoda-com/opentelemetry-logs-go/sdk/logs" otelpyroscope "github.com/grafana/otel-profiling-go" "github.com/grafana/pyroscope-go" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + semconv "go.opentelemetry.io/otel/semconv/v1.25.0" ) type Client struct { @@ -76,9 +76,9 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { return nil, err } - meticExporter, err := otlpmetricgrpc.New(ctx, - otlpmetricgrpc.WithEndpoint(endpoint), - otlpmetricgrpc.WithRetry(otlpmetricgrpc.RetryConfig{ + meticExporter, err := otlpmetrichttp.New(ctx, + otlpmetrichttp.WithEndpoint(endpoint), + otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{ Enabled: false, }), ) @@ -118,8 +118,8 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { logExporter, err := otlplogs.NewExporter(ctx, otlplogs.WithClient( - otlplogsgrpc.NewClient( - otlplogsgrpc.WithEndpoint(endpoint), + otlplogshttp.NewClient( + otlplogshttp.WithEndpoint(endpoint), ), ), ) @@ -140,34 +140,34 @@ func Setup(ctx context.Context, endpoint string) (*Client, error) { // recreate telemetry logger client.log = rlog.Component("telemetry") - runtime.SetMutexProfileFraction(5) - runtime.SetBlockProfileRate(5) - _, err = pyroscope.Start(pyroscope.Config{ - ApplicationName: appName, - // replace this with the address of pyroscope server - ServerAddress: "https://pyroscope.kmsign.ru", - // you can disable logging by setting this to nil - Logger: &pyroscopeLogger{ - log: client.log.WithComponent("pyroscope"), - }, - ProfileTypes: []pyroscope.ProfileType{ - // these profile types are enabled by default: - pyroscope.ProfileCPU, - pyroscope.ProfileAllocObjects, - pyroscope.ProfileAllocSpace, - pyroscope.ProfileInuseObjects, - pyroscope.ProfileInuseSpace, - // these profile types are optional: - // pyroscope.ProfileGoroutines, - // pyroscope.ProfileMutexCount, - // pyroscope.ProfileMutexDuration, - // pyroscope.ProfileBlockCount, - // pyroscope.ProfileBlockDuration, - }, - }) - if err != nil { - return client, nil - } + // runtime.SetMutexProfileFraction(5) + // runtime.SetBlockProfileRate(5) + // _, err = pyroscope.Start(pyroscope.Config{ + // ApplicationName: appName, + // // replace this with the address of pyroscope server + // ServerAddress: "https://pyroscope.kmsign.ru", + // // you can disable logging by setting this to nil + // Logger: &pyroscopeLogger{ + // log: client.log.WithComponent("pyroscope"), + // }, + // ProfileTypes: []pyroscope.ProfileType{ + // // these profile types are enabled by default: + // pyroscope.ProfileCPU, + // pyroscope.ProfileAllocObjects, + // pyroscope.ProfileAllocSpace, + // pyroscope.ProfileInuseObjects, + // pyroscope.ProfileInuseSpace, + // // these profile types are optional: + // // pyroscope.ProfileGoroutines, + // // pyroscope.ProfileMutexCount, + // // pyroscope.ProfileMutexDuration, + // // pyroscope.ProfileBlockCount, + // // pyroscope.ProfileBlockDuration, + // }, + // }) + // if err != nil { + // return client, nil + // } return client, nil }