/go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:27.346Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:28.156Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:31:28.157Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:28.157Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.194ms", "json": "34.569µs", "db": "540ns", "time_taken": "1.303157ms"} 2026-06-24T17:31:35.010Z INFO blobber/worker.go:51 success to refresh blobber settings from chain 2026-06-24T17:31:38.158Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:31:38.159Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:38.159Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.701441ms", "json": "65.749µs", "db": "930ns", "time_taken": "1.861358ms"} 2026-06-24T17:31:48.160Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:31:48.162Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:48.162Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.813638ms", "json": "39.31µs", "db": "550ns", "time_taken": "1.954676ms"} 2026-06-24T17:31:58.163Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:31:58.166Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:58.166Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.100286ms", "json": "74.719µs", "db": "1.36µs", "time_taken": "3.286793ms"} 2026-06-24T17:31:59.317Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "781015c394c2edcf9d21a9b0ea60bfb3d0dac92fc7b1f3f5b9e6e140a49017ac", "start": "2026-06-24T17:31:53.292Z", "end": "2026-06-24T17:31:59.317Z", "duration": "6.02570327s"} 2026-06-24T17:32:08.166Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:08.168Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:08.168Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.548113ms", "json": "30.6µs", "db": "600ns", "time_taken": "1.648022ms"} 2026-06-24T17:32:18.169Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:18.172Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:18.172Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.945959ms", "json": "51.409µs", "db": "570ns", "time_taken": "3.076106ms"} 2026-06-24T17:32:28.172Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:28.174Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:28.174Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.018455ms", "json": "101.188µs", "db": "1.51µs", "time_taken": "2.243321ms"} 2026-06-24T17:32:38.174Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:38.176Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "WoMK3A5P6mad7UZDn8Afya"} 2026-06-24T17:32:38.176Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:38.176Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.72144ms", "json": "51.809µs", "db": "1.03µs", "time_taken": "1.833118ms"} 2026-06-24T17:32:38.177Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "BgHNpeYznzASk8JPkGMT94"} 2026-06-24T17:32:38.178Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "V9QXe8r2QfUQ2Xd4mP2TdL"} 2026-06-24T17:32:48.177Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:48.179Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:48.179Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.590613ms", "json": "71.789µs", "db": "1.17µs", "time_taken": "1.74062ms"} 2026-06-24T17:32:57.310Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "f3f7e33c5dca024a5c429822d00363576caf5fc7ae600f63a1f6741e2a14651a", "start": "2026-06-24T17:32:53.292Z", "end": "2026-06-24T17:32:57.310Z", "duration": "4.018561246s"} 2026-06-24T17:32:58.179Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:32:58.181Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:58.181Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.637531ms", "json": "52.989µs", "db": "820ns", "time_taken": "1.760479ms"} 2026-06-24T17:32:58.436Z ERROR handler/object_operation_handler.go:1637 Error file reference {"error": "record not found"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.(*StorageHandler).CreateDir /0chain/code/go/0chain.net/blobbercore/handler/object_operation_handler.go:1637 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CreateDirHandler /0chain/code/go/0chain.net/blobbercore/handler/handler.go:1398 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11.1 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:327 github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore.(*postgresStore).WithNewTransaction /0chain/code/go/0chain.net/blobbercore/datastore/postgres.go:110 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:326 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToJSONResponse.func12 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func72 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.517Z ERROR common/request_form.go:23 TryParseForm: ParseMultipartForm {"error": "request Content-Type isn't multipart/form-data"} github.com/0chain/blobber/code/go/0chain.net/core/common.TryParseForm /0chain/code/go/0chain.net/core/common/request_form.go:23 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithTxHandler.func61 /0chain/code/go/0chain.net/blobbercore/handler/context.go:146 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func95 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.519Z INFO writemarker/mutex.go:51 Locking write marker {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073", "connection_id": "arkZc9z8TfGaZRnUgpHtc9"} 2026-06-24T17:32:58.519Z INFO handler/handler_writemarker.go:56 Lock write marker result {"result": {"status":2,"created_at":1782322378}} 2026-06-24T17:32:58.562Z INFO handler/handler_common.go:147 Locking allocation {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.563Z INFO handler/object_operation_handler.go:907 commit_write {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.564Z ERROR handler/handler_common.go:65 commitHandler_request_failed {"error": "latest_write_marker_failed: Latest write marker is in failed state"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.commitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:65 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CommitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:746 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:156 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.564Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:33:08.181Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:08.183Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:08.183Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.849618ms", "json": "63.259µs", "db": "910ns", "time_taken": "1.983935ms"} 2026-06-24T17:33:18.184Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:18.187Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:18.187Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.500367ms", "json": "63.909µs", "db": "860ns", "time_taken": "2.665674ms"} 2026-06-24T17:33:28.187Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:28.194Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:28.194Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "7.359602ms", "json": "67.129µs", "db": "730ns", "time_taken": "7.718736ms"} 2026-06-24T17:33:38.196Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:38.197Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:38.197Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.363436ms", "json": "126.998µs", "db": "1.72µs", "time_taken": "1.598353ms"} 2026-06-24T17:33:48.197Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:48.199Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:48.199Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.285978ms", "json": "47.989µs", "db": "620ns", "time_taken": "1.410685ms"} 2026-06-24T17:33:57.314Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "1a623cf49cae750d5d58bbc75cad9228acc9c116127ceddeab92524e8aded599", "start": "2026-06-24T17:33:53.293Z", "end": "2026-06-24T17:33:57.314Z", "duration": "4.021629056s"} 2026-06-24T17:33:58.199Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:33:58.201Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:58.201Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.553823ms", "json": "62.299µs", "db": "800ns", "time_taken": "1.70519ms"} 2026-06-24T17:34:08.174Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcY7ZLPr4SVWWo5gdYuzc6"} 2026-06-24T17:34:08.176Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcDsLpSSKs9KUMZfGSUnrT"} 2026-06-24T17:34:08.177Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "LgmnePpNAayfhboTSoqkZk"} 2026-06-24T17:34:08.202Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:34:08.203Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:08.203Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.322597ms", "json": "53.219µs", "db": "570ns", "time_taken": "1.451045ms"} 2026-06-24T17:34:18.204Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"af908d6f08edf09372940b1f1ce62268534e41d959489400efd890ff7e24bd6b","from":"7175187","limit":"20"}} 2026-06-24T17:34:18.208Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:34:18.208Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.864863ms", "json": "44.72µs", "db": "770ns", "time_taken": "4.005711ms"}