net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:27.336Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:31:28.635Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:31:28.637Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:28.637Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "2.056694ms", "json": "43.959µs", "db": "660ns", "time_taken": "2.221571ms"} 2026-06-24T17:31:38.637Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:31:38.640Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:38.640Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "3.135476ms", "json": "56.309µs", "db": "520ns", "time_taken": "3.259543ms"} 2026-06-24T17:31:43.216Z INFO blobber/worker.go:51 success to refresh blobber settings from chain 2026-06-24T17:31:48.641Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:31:48.643Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:48.643Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.515184ms", "json": "81.418µs", "db": "1.57µs", "time_taken": "1.666221ms"} 2026-06-24T17:31:58.644Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:31:58.645Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:31:58.645Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.090071ms", "json": "35.78µs", "db": "770ns", "time_taken": "1.194699ms"} 2026-06-24T17:32:07.019Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "db15e93faad16feef6d6fbcdb126ae36f7990360ee9dd1c9c40e46310cf33214", "start": "2026-06-24T17:32:04.005Z", "end": "2026-06-24T17:32:07.019Z", "duration": "3.0146049s"} 2026-06-24T17:32:08.645Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:08.650Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:08.650Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "5.19007ms", "json": "60.069µs", "db": "770ns", "time_taken": "5.332817ms"} 2026-06-24T17:32:18.651Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:18.653Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:18.653Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.72005ms", "json": "51.719µs", "db": "910ns", "time_taken": "1.846668ms"} 2026-06-24T17:32:18.683Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "V9QXe8r2QfUQ2Xd4mP2TdL"} 2026-06-24T17:32:18.684Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "WoMK3A5P6mad7UZDn8Afya"} 2026-06-24T17:32:18.684Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "BgHNpeYznzASk8JPkGMT94"} 2026-06-24T17:32:28.654Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:28.664Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:28.665Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "10.345451ms", "json": "114.238µs", "db": "710ns", "time_taken": "10.620856ms"} 2026-06-24T17:32:38.665Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:38.667Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:38.667Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.404565ms", "json": "44.979µs", "db": "640ns", "time_taken": "1.512603ms"} 2026-06-24T17:32:48.668Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:48.669Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:48.669Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.500724ms", "json": "56.279µs", "db": "1.06µs", "time_taken": "1.634582ms"} 2026-06-24T17:32:58.436Z ERROR handler/object_operation_handler.go:1637 Error file reference {"error": "record not found"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.(*StorageHandler).CreateDir /0chain/code/go/0chain.net/blobbercore/handler/object_operation_handler.go:1637 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CreateDirHandler /0chain/code/go/0chain.net/blobbercore/handler/handler.go:1398 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11.1 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:327 github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore.(*postgresStore).WithNewTransaction /0chain/code/go/0chain.net/blobbercore/datastore/postgres.go:110 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithConnection.func11 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:326 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToJSONResponse.func12 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func72 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.520Z ERROR common/request_form.go:23 TryParseForm: ParseMultipartForm {"error": "request Content-Type isn't multipart/form-data"} github.com/0chain/blobber/code/go/0chain.net/core/common.TryParseForm /0chain/code/go/0chain.net/core/common/request_form.go:23 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithTxHandler.func61 /0chain/code/go/0chain.net/blobbercore/handler/context.go:146 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByGeneralRL.RateLimit.func95 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.522Z INFO writemarker/mutex.go:51 Locking write marker {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073", "connection_id": "arkZc9z8TfGaZRnUgpHtc9"} 2026-06-24T17:32:58.522Z INFO handler/handler_writemarker.go:56 Lock write marker result {"result": {"status":2,"created_at":1782322378}} 2026-06-24T17:32:58.569Z INFO handler/handler_common.go:147 Locking allocation {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.569Z INFO handler/object_operation_handler.go:907 commit_write {"allocation_id": "5dbeee689d918c72dca551dce7601f88094791c26cb400efdfcadf9b442cf073"} 2026-06-24T17:32:58.571Z ERROR handler/handler_common.go:65 commitHandler_request_failed {"error": "latest_write_marker_failed: Latest write marker is in failed state"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.commitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:65 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.CommitHandlerV2 /0chain/code/go/0chain.net/blobbercore/handler/handler.go:746 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:156 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.571Z ERROR handler/handler_common.go:171 Error in handling the request.latest_write_marker_failed: Latest write marker is in failed state github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.WithStatusConnectionForWM.func15 /0chain/code/go/0chain.net/blobbercore/handler/handler_common.go:171 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.ToStatusCode.func16 /0chain/code/go/0chain.net/core/common/handler.go:178 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.setupHandlers.RateLimitByCommmitRL.RateLimit.func74 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-24T17:32:58.670Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:32:58.671Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:32:58.671Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.258788ms", "json": "33.99µs", "db": "700ns", "time_taken": "1.380876ms"} 2026-06-24T17:33:07.021Z INFO handler/worker.go:39 success to send heartbeat {"txn_hash": "162c22fe57a851ed8a4395c604bd0f355ffdbf708e27b3ed0274b42b561d4ea9", "start": "2026-06-24T17:33:04.005Z", "end": "2026-06-24T17:33:07.021Z", "duration": "3.015882961s"} 2026-06-24T17:33:08.672Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:08.704Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:08.704Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "31.6377ms", "json": "68.449µs", "db": "620ns", "time_taken": "31.802488ms"} 2026-06-24T17:33:18.705Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:18.707Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:18.707Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.556053ms", "json": "52.459µs", "db": "830ns", "time_taken": "1.72053ms"} 2026-06-24T17:33:28.708Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:28.730Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:28.730Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "22.089256ms", "json": "91.739µs", "db": "720ns", "time_taken": "22.296663ms"} 2026-06-24T17:33:38.731Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:38.733Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:38.733Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.340087ms", "json": "50.4µs", "db": "840ns", "time_taken": "1.473615ms"} 2026-06-24T17:33:48.683Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcDsLpSSKs9KUMZfGSUnrT"} 2026-06-24T17:33:48.684Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "LgmnePpNAayfhboTSoqkZk"} 2026-06-24T17:33:48.685Z INFO handler/worker.go:90 Deleting temp files for the connection {"connection": "tcY7ZLPr4SVWWo5gdYuzc6"} 2026-06-24T17:33:48.733Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:48.735Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:48.735Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.73166ms", "json": "68.149µs", "db": "1.11µs", "time_taken": "1.866047ms"} 2026-06-24T17:33:58.736Z INFO challenge/challenge.go:54 [challenge]sync:pull {"params": {"blobber":"2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18","from":"7175184","limit":"20"}} 2026-06-24T17:33:58.737Z INFO challenge/challenge.go:87 challenges_from_chain {"challenges": 0, "challenge_ids": []} 2026-06-24T17:33:58.737Z INFO challenge/challenge.go:99 [challenge]elapsed:pull {"count": 0, "download": "1.353427ms", "json": "51.479µs", "db": "1.39µs", "time_taken": "1.493204ms"}