/go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:27.350Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:28.300Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:31:28.302Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:28.302Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.773869ms", "json": "64.029µs", "db": "1.32µs", "time_taken": "1.930477ms"} 2026-06-24T17:31:38.303Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:31:38.305Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:38.305Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.842568ms", "json": "66.969µs", "db": "1.29µs", "time_taken": "1.972736ms"} 2026-06-24T17:31:48.306Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:31:48.308Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:48.308Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.717191ms", "json": "49.859µs", "db": "660ns", "time_taken": "1.882858ms"} 2026-06-24T17:31:52.668Z INFO blobber/worker.go:51 success to refresh blobber settings from chain 2026-06-24T17:31:58.308Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:31:58.310Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:58.310Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.810819ms", "json": "75.438µs", "db": "1.1µs", "time_taken": "1.972785ms"} 2026-06-24T17:32:08.311Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:08.313Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:08.313Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.830018ms", "json": "64.869µs", "db": "1.06µs", "time_taken": "1.978916ms"} 2026-06-24T17:32:18.039Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "d000d55e0410a9306c6763a3cb2845a67965d7977874bc38c599a235d35cff99", "start": "2026-06-24T17:32:15.011Z", "end": "2026-06-24T17:32:18.039Z", "duration": "3.027913369s"} 2026-06-24T17:32:18.314Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:18.318Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:18.318Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.518959ms", "json": "255.515µs", "db": "1.95µs", "time_taken": "3.929662ms"} 2026-06-24T17:32:28.319Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:28.321Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:28.321Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.649212ms", "json": "54.939µs", "db": "620ns", "time_taken": "1.863528ms"} 2026-06-24T17:32:29.914Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "WoMK3A5P6mad7UZDn8Afya"} 2026-06-24T17:32:29.916Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "BgHNpeYznzASk8JPkGMT94"} 2026-06-24T17:32:29.917Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "V9QXe8r2QfUQ2Xd4mP2TdL"} 2026-06-24T17:32:38.321Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:38.325Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:38.325Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.865043ms", "json": "50.859µs", "db": "670ns", "time_taken": "4.01596ms"} 2026-06-24T17:32:48.325Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:48.327Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:48.327Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.539063ms", "json": "34.049µs", "db": "590ns", "time_taken": "1.671711ms"} 2026-06-24T17:32:58.327Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:32:58.329Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:58.329Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.458514ms", "json": "39.14µs", "db": "610ns", "time_taken": "1.598763ms"} 2026-06-24T17:32:58.436Z ERROR handler/object_operation_handler.go:1637 Error file reference {"error": "record not found"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.(*StorageHandler).CreateDir /0chain/code/go/0chain.net/blobbercore/handler/object_operation_handler.go:1637 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CreateDirHandler /0chain/code/go/0chain.net/blobbercore/handler/handler.go:1398 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11.1 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:327 github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore.(*postgresStore).WithNewTransaction /0chain/code/go/0chain.net/blobbercore/datastore/postgres.go:110 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:326 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToJSONResponse.func12 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func72 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.505Z ERROR common/request_form.go:23 TryParseForm: ParseMultipartForm {"error": "request Content-Type isn't multipart/form-data"} github.com/0chain/blobber/code/go/0chain.net/core/common.TryParseForm /0chain/code/go/0chain.net/core/common/request_form.go:23 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithTxHandler.func61 /0chain/code/go/0chain.net/blobbercore/handler/context.go:146 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func95 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.507Z INFO writemarker/mutex.go:51 Locking write marker {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073", "connection_id": "arkZc9z8TfGaZRnUgpHtc9"} 2026-06-24T17:32:58.507Z INFO handler/handler_writemarker.go:56 Lock write marker result {"result": {"status":2,"created_at":1782322378}} 2026-06-24T17:32:58.550Z INFO handler/handler_common.go:147 Locking allocation {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.550Z INFO handler/object_operation_handler.go:907 commit_write {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.552Z ERROR handler/handler_common.go:65 commitHandler_request_failed {"error": "latest_write_marker_failed: Latest write marker is in failed state"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.commitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:65 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CommitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:746 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:156 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.552Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:33:08.330Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:08.332Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:08.332Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.777429ms", "json": "66.949µs", "db": "1.03µs", "time_taken": "1.966936ms"} 2026-06-24T17:33:18.039Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "58fa26c7e9f528db713d1ad743d5884adda42c82f18f14ee33742af1fa0d84a6", "start": "2026-06-24T17:33:15.011Z", "end": "2026-06-24T17:33:18.039Z", "duration": "3.02804536s"} 2026-06-24T17:33:18.333Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:18.337Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:18.337Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.723776ms", "json": "58.229µs", "db": "1.48µs", "time_taken": "3.919882ms"} 2026-06-24T17:33:28.337Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:28.339Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:28.339Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.71779ms", "json": "77.589µs", "db": "1.15µs", "time_taken": "1.874398ms"} 2026-06-24T17:33:38.340Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:38.342Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:38.342Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.73586ms", "json": "57.659µs", "db": "1.08µs", "time_taken": "1.872188ms"} 2026-06-24T17:33:48.343Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:48.344Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:48.344Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.76038ms", "json": "68.399µs", "db": "1.029µs", "time_taken": "1.895227ms"} 2026-06-24T17:33:58.345Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:33:58.346Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:58.346Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.749639ms", "json": "32.78µs", "db": "850ns", "time_taken": "1.861588ms"} 2026-06-24T17:33:59.913Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcDsLpSSKs9KUMZfGSUnrT"} 2026-06-24T17:33:59.916Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcY7ZLPr4SVWWo5gdYuzc6"} 2026-06-24T17:33:59.917Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "LgmnePpNAayfhboTSoqkZk"} 2026-06-24T17:34:08.347Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:34:08.349Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:08.349Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.433655ms", "json": "53.579µs", "db": "610ns", "time_taken": "1.561583ms"} 2026-06-24T17:34:18.349Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0","from":"7175178","limit":"20"}} 2026-06-24T17:34:18.351Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:18.351Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.779609ms", "json": "55.469µs", "db": "830ns", "time_taken": "1.912397ms"} 2026-06-24T17:34:19.049Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "96404072ef831eaba1d03e048afd434e43c8cf569abcaf020cf46aa267db0aa3", "start": "2026-06-24T17:34:15.014Z", "end": "2026-06-24T17:34:19.049Z", "duration": "4.035252711s"}