net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:27.331Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:28.625Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:31:28.626Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:28.626Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.273218ms", "json": "49.459µs", "db": "690ns", "time_taken": "1.384686ms"} 2026-06-24T17:31:38.628Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:31:38.629Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:38.629Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.431475ms", "json": "27.179µs", "db": "400ns", "time_taken": "1.511864ms"} 2026-06-24T17:31:43.224Z INFO blobber/worker.go:51 success to refresh blobber settings from chain 2026-06-24T17:31:48.630Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:31:48.631Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:48.631Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.430225ms", "json": "53.739µs", "db": "800ns", "time_taken": "1.564013ms"} 2026-06-24T17:31:58.631Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:31:58.633Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:58.633Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.638952ms", "json": "37.999µs", "db": "580ns", "time_taken": "1.773749ms"} 2026-06-24T17:32:07.005Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "d699b6d0f1646845a18b4081210deb8d12be580482910d23fd8034761372d89d", "start": "2026-06-24T17:32:03.990Z", "end": "2026-06-24T17:32:07.005Z", "duration": "3.015262669s"} 2026-06-24T17:32:08.634Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:08.637Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:08.637Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.478009ms", "json": "29.42µs", "db": "1.19µs", "time_taken": "3.601857ms"} 2026-06-24T17:32:18.637Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:18.639Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:18.639Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.495495ms", "json": "36.949µs", "db": "590ns", "time_taken": "1.601032ms"} 2026-06-24T17:32:18.649Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "WoMK3A5P6mad7UZDn8Afya"} 2026-06-24T17:32:18.650Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "BgHNpeYznzASk8JPkGMT94"} 2026-06-24T17:32:18.652Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "V9QXe8r2QfUQ2Xd4mP2TdL"} 2026-06-24T17:32:28.640Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:28.644Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:28.644Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.779774ms", "json": "85.359µs", "db": "1.27µs", "time_taken": "3.949632ms"} 2026-06-24T17:32:38.644Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:38.646Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:38.646Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.382596ms", "json": "38.999µs", "db": "670ns", "time_taken": "1.488714ms"} 2026-06-24T17:32:48.646Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:48.647Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:48.647Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.443475ms", "json": "70.059µs", "db": "830ns", "time_taken": "1.602762ms"} 2026-06-24T17:32:58.437Z ERROR handler/object_operation_handler.go:1637 Error file reference {"error": "record not found"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.(*StorageHandler).CreateDir /0chain/code/go/0chain.net/blobbercore/handler/object_operation_handler.go:1637 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CreateDirHandler /0chain/code/go/0chain.net/blobbercore/handler/handler.go:1398 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11.1 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:327 github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore.(*postgresStore).WithNewTransaction /0chain/code/go/0chain.net/blobbercore/datastore/postgres.go:110 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:326 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToJSONResponse.func12 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func72 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.508Z ERROR common/request_form.go:23 TryParseForm: ParseMultipartForm {"error": "request Content-Type isn't multipart/form-data"} github.com/0chain/blobber/code/go/0chain.net/core/common.TryParseForm /0chain/code/go/0chain.net/core/common/request_form.go:23 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithTxHandler.func61 /0chain/code/go/0chain.net/blobbercore/handler/context.go:146 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func95 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.510Z INFO writemarker/mutex.go:51 Locking write marker {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073", "connection_id": "arkZc9z8TfGaZRnUgpHtc9"} 2026-06-24T17:32:58.510Z INFO handler/handler_writemarker.go:56 Lock write marker result {"result": {"status":2,"created_at":1782322378}} 2026-06-24T17:32:58.556Z INFO handler/handler_common.go:147 Locking allocation {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.556Z INFO handler/object_operation_handler.go:907 commit_write {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.557Z ERROR handler/handler_common.go:65 commitHandler_request_failed {"error": "latest_write_marker_failed: Latest write marker is in failed state"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.commitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:65 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CommitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:746 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:156 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.557Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.648Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:32:58.649Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:58.649Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.601352ms", "json": "55.649µs", "db": "700ns", "time_taken": "1.73568ms"} 2026-06-24T17:33:07.006Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "0e5347ef7d1a8343702f2b7afaeea2432353d4631a1855988354c45950529787", "start": "2026-06-24T17:33:03.990Z", "end": "2026-06-24T17:33:07.006Z", "duration": "3.016361693s"} 2026-06-24T17:33:08.650Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:08.653Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:08.653Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.926229ms", "json": "76.979µs", "db": "950ns", "time_taken": "3.081007ms"} 2026-06-24T17:33:18.654Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:18.657Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:18.657Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.387759ms", "json": "64.649µs", "db": "960ns", "time_taken": "2.510117ms"} 2026-06-24T17:33:28.657Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:28.659Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:28.660Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.216252ms", "json": "96.578µs", "db": "690ns", "time_taken": "2.546936ms"} 2026-06-24T17:33:38.660Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:38.662Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:38.662Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.374206ms", "json": "57.579µs", "db": "1.01µs", "time_taken": "1.491594ms"} 2026-06-24T17:33:48.650Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcY7ZLPr4SVWWo5gdYuzc6"} 2026-06-24T17:33:48.651Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcDsLpSSKs9KUMZfGSUnrT"} 2026-06-24T17:33:48.652Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "LgmnePpNAayfhboTSoqkZk"} 2026-06-24T17:33:48.662Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:48.663Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:48.663Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.573602ms", "json": "52.129µs", "db": "640ns", "time_taken": "1.674721ms"} 2026-06-24T17:33:58.664Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"a9e877b71effeb6b8e3687a8fe1244967c13e750960fe97f1c2e5ecb7358f8d8","from":"7175186","limit":"20"}} 2026-06-24T17:33:58.665Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:58.665Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.456775ms", "json": "47.039µs", "db": "780ns", "time_taken": "1.615592ms"}