/go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:27.326Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:28.632Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:31:28.634Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:28.634Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.69371ms", "json": "48.739µs", "db": "680ns", "time_taken": "1.848008ms"} 2026-06-24T17:31:38.634Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:31:38.636Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:38.636Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.267298ms", "json": "44.029µs", "db": "690ns", "time_taken": "1.371586ms"} 2026-06-24T17:31:48.636Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:31:48.637Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:48.637Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.349997ms", "json": "29.399µs", "db": "400ns", "time_taken": "1.457914ms"} 2026-06-24T17:31:52.659Z INFO blobber/worker.go:51 success to refresh blobber settings from chain 2026-06-24T17:31:58.638Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:31:58.639Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:58.639Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.12826ms", "json": "54.429µs", "db": "510ns", "time_taken": "1.250088ms"} 2026-06-24T17:32:08.640Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:08.641Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:08.641Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.123131ms", "json": "30.579µs", "db": "410ns", "time_taken": "1.200679ms"} 2026-06-24T17:32:18.264Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "08aa224d5cac554ade518fe0cbf8a65e8e0a3ba770c7019ccf654ae7e93756a8", "start": "2026-06-24T17:32:14.246Z", "end": "2026-06-24T17:32:18.264Z", "duration": "4.018454646s"} 2026-06-24T17:32:18.641Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:18.643Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:18.643Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.327579ms", "json": "139.968µs", "db": "1.93µs", "time_taken": "2.562706ms"} 2026-06-24T17:32:28.645Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:28.652Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:28.652Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "6.653084ms", "json": "104.848µs", "db": "1.02µs", "time_taken": "6.87851ms"} 2026-06-24T17:32:29.163Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "WoMK3A5P6mad7UZDn8Afya"} 2026-06-24T17:32:29.164Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "BgHNpeYznzASk8JPkGMT94"} 2026-06-24T17:32:29.165Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "V9QXe8r2QfUQ2Xd4mP2TdL"} 2026-06-24T17:32:38.653Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:38.655Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:38.655Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.13152ms", "json": "43.829µs", "db": "620ns", "time_taken": "1.238338ms"} 2026-06-24T17:32:48.655Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:48.656Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:48.656Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.340827ms", "json": "36.95µs", "db": "630ns", "time_taken": "1.437846ms"} 2026-06-24T17:32:58.435Z ERROR handler/object_operation_handler.go:1637 Error file reference {"error": "record not found"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.(*StorageHandler).CreateDir /0chain/code/go/0chain.net/blobbercore/handler/object_operation_handler.go:1637 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CreateDirHandler /0chain/code/go/0chain.net/blobbercore/handler/handler.go:1398 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11.1 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:327 github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore.(*postgresStore).WithNewTransaction /0chain/code/go/0chain.net/blobbercore/datastore/postgres.go:110 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:326 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToJSONResponse.func12 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func72 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.518Z ERROR common/request_form.go:23 TryParseForm: ParseMultipartForm {"error": "request Content-Type isn't multipart/form-data"} github.com/0chain/blobber/code/go/0chain.net/core/common.TryParseForm /0chain/code/go/0chain.net/core/common/request_form.go:23 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithTxHandler.func61 /0chain/code/go/0chain.net/blobbercore/handler/context.go:146 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func95 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.519Z INFO writemarker/mutex.go:51 Locking write marker {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073", "connection_id": "arkZc9z8TfGaZRnUgpHtc9"} 2026-06-24T17:32:58.519Z INFO handler/handler_writemarker.go:56 Lock write marker result {"result": {"status":2,"created_at":1782322378}} 2026-06-24T17:32:58.544Z INFO handler/handler_common.go:147 Locking allocation {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.544Z INFO handler/object_operation_handler.go:907 commit_write {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.546Z ERROR handler/handler_common.go:65 commitHandler_request_failed {"error": "latest_write_marker_failed: Latest write marker is in failed state"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.commitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:65 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CommitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:746 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:156 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.546Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.657Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:32:58.659Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:58.659Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.466914ms", "json": "47.529µs", "db": "1.24µs", "time_taken": "1.585822ms"} 2026-06-24T17:33:08.659Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:08.665Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:08.665Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "5.804619ms", "json": "75.119µs", "db": "760ns", "time_taken": "5.983597ms"} 2026-06-24T17:33:17.262Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "70583f3d74a6194128ac08ccdd370005e51419b516c08ff96e61f81780cea1e5", "start": "2026-06-24T17:33:14.246Z", "end": "2026-06-24T17:33:17.262Z", "duration": "3.015089505s"} 2026-06-24T17:33:18.666Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:18.667Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:18.667Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.781999ms", "json": "61.049µs", "db": "830ns", "time_taken": "1.921186ms"} 2026-06-24T17:33:28.668Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:28.675Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:28.675Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "6.952869ms", "json": "74.069µs", "db": "660ns", "time_taken": "7.105717ms"} 2026-06-24T17:33:38.675Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:38.677Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:38.677Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.72853ms", "json": "78.288µs", "db": "1.01µs", "time_taken": "1.879927ms"} 2026-06-24T17:33:48.677Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:48.681Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:48.681Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.131795ms", "json": "53.739µs", "db": "940ns", "time_taken": "3.258343ms"} 2026-06-24T17:33:58.681Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:33:58.683Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:58.683Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.590622ms", "json": "76.519µs", "db": "1.38µs", "time_taken": "1.7345ms"} 2026-06-24T17:33:59.163Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcY7ZLPr4SVWWo5gdYuzc6"} 2026-06-24T17:33:59.168Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcDsLpSSKs9KUMZfGSUnrT"} 2026-06-24T17:33:59.170Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "LgmnePpNAayfhboTSoqkZk"} 2026-06-24T17:34:08.684Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:34:08.686Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:08.686Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.557143ms", "json": "87.489µs", "db": "840ns", "time_taken": "1.76276ms"} 2026-06-24T17:34:18.264Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "b145ea9e35ecf1d0e5d589ba3d295a0521a5e06e76e4a5db97c29dee622bb65d", "start": "2026-06-24T17:34:14.247Z", "end": "2026-06-24T17:34:18.264Z", "duration": "4.017121966s"} 2026-06-24T17:34:18.686Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"14511a851a7c5d082cde81cc32817bfa8515f1232881ebbbb8e3ccc884ec7ed7","from":"7175185","limit":"20"}} 2026-06-24T17:34:18.688Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:18.688Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.417796ms", "json": "50.409µs", "db": "750ns", "time_taken": "1.542544ms"}