electrumx reports an error in the container about electrumx

[SomberNight]

Can you reproduce this still?

Would be nice if you could get the value of n.

electrumx/electrumx/server/block_processor.py

Line 547 in bf43035

assert n == 0

just change that line to

assert n == 0, n

[Jamiejoin]

@SomberNight Thank you very much for your reply，I can cooperate with you to modify the configuration on my server，Because of the high configuration of my server, the disk and memory are very large, I run two electrumx services on this server, one is in the container, the other is directly running on the server, pointing to different storage disks, but now running After a few days both have the same error

ubuntu-PowerEdge-R740 electrumx_server[129303]: ERROR:electrumx:ElectrumX server terminated abnormally
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]: Traceback (most recent call last):
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/bin/electrumx_server", line 35, in main
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     asyncio.run(controller.run())
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/asyncio/runners.py", line 43, in run
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     return loop.run_until_complete(main)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/asyncio/base_events.py", line 587, in run_until_complete
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     return future.result()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/lib/server_base.py", line 129, in run
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await server_task
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/lib/server_base.py", line 102, in serve
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.serve(shutdown_event)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/controller.py", line 134, in serve
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await group.spawn(wait_for_catchup())
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/aiorpcx/curio.py", line 242, in __aexit__
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.join()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/aiorpcx/curio.py", line 211, in join
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     raise task.exception()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 702, in fetch_and_process_blocks
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await group.spawn(self._process_prefetched_blocks())
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/aiorpcx/curio.py", line 242, in __aexit__
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.join()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/aiorpcx/curio.py", line 211, in join
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     raise task.exception()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 663, in _process_prefetched_blocks
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.check_and_advance_blocks(blocks)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 240, in check_and_advance_blocks
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.reorg_chain()
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 282, in reorg_chain
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 212, in run_in_thread_with_lock
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     return await asyncio.shield(run_in_thread_locked())
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 211, in run_in_thread_locked
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     return await run_in_thread(func, *args)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/aiorpcx/curio.py", line 68, in run_in_thread
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     return await get_event_loop().run_in_executor(None, func, *args)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/concurrent/futures/thread.py", line 57, in run
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     result = self.fn(*self.args, **self.kwargs)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 500, in backup_blocks
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     self.backup_txs(block.transactions, is_unspendable)
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:   File "/usr/local/lib/python3.7/site-packages/electrumx/server/block_processor.py", line 547, in backup_txs
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]:     assert n == 0, n
Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]: AssertionError: 12642

[Jamiejoin]

This should be a BUG, I now restore the data to the backup 1 month ago, and restart electrumx, no errors are found, I am testing. Try to restart the electrumx server you are running, you may have the same problem as me

[SomberNight]

Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]: AssertionError: 12642

That number n, should normally be a multiple of undo_entry_len (==24).

Did you shut down e-x uncleanly right before this happened?

[Jamiejoin]

Nov 13 17:48:05 ubuntu-PowerEdge-R740 electrumx_server[129303]: AssertionError: 12642

That number n, should normally be a multiple of undo_entry_len (==24).

Did you shut down e-x uncleanly right before this happened?

I think what you said is right, I plan to try a few times

[Jamiejoin]

@SomberNight It is strange that the following error still appears during operation. Is there any way to fix it?

INFO:Controller:shutdown complete
ERROR:electrumx:ElectrumX server terminated abnormally
Traceback (most recent call last):
  File "/electrumx/electrumx_server", line 35, in main
    asyncio.run(controller.run())
  File "/usr/local/lib/python3.7/asyncio/runners.py", line 43, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.7/asyncio/base_events.py", line 587, in run_until_complete
    return future.result()
  File "/electrumx/electrumx/lib/server_base.py", line 129, in run
    await server_task
  File "/electrumx/electrumx/lib/server_base.py", line 102, in serve
    await self.serve(shutdown_event)
  File "/electrumx/electrumx/server/controller.py", line 134, in serve
    await group.spawn(wait_for_catchup())
  File "/usr/local/lib/python3.7/site-packages/aiorpcX-0.18.4-py3.7.egg/aiorpcx/curio.py", line 242, in __aexit__
    await self.join()
  File "/usr/local/lib/python3.7/site-packages/aiorpcX-0.18.4-py3.7.egg/aiorpcx/curio.py", line 211, in join
    raise task.exception()
  File "/electrumx/electrumx/server/block_processor.py", line 702, in fetch_and_process_blocks
    await group.spawn(self._process_prefetched_blocks())
  File "/usr/local/lib/python3.7/site-packages/aiorpcX-0.18.4-py3.7.egg/aiorpcx/curio.py", line 242, in __aexit__
    await self.join()
  File "/usr/local/lib/python3.7/site-packages/aiorpcX-0.18.4-py3.7.egg/aiorpcx/curio.py", line 211, in join
    raise task.exception()
  File "/electrumx/electrumx/server/block_processor.py", line 663, in _process_prefetched_blocks
    await self.check_and_advance_blocks(blocks)
  File "/electrumx/electrumx/server/block_processor.py", line 240, in check_and_advance_blocks
    await self.reorg_chain()
  File "/electrumx/electrumx/server/block_processor.py", line 282, in reorg_chain
    await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
  File "/electrumx/electrumx/server/block_processor.py", line 212, in run_in_thread_with_lock
    return await asyncio.shield(run_in_thread_locked())
  File "/electrumx/electrumx/server/block_processor.py", line 211, in run_in_thread_locked
    return await run_in_thread(func, *args)
  File "/usr/local/lib/python3.7/site-packages/aiorpcX-0.18.4-py3.7.egg/aiorpcx/curio.py", line 68, in run_in_thread
    return await get_event_loop().run_in_executor(None, func, *args)
  File "/usr/local/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/electrumx/electrumx/server/block_processor.py", line 500, in backup_blocks
    self.backup_txs(block.transactions, is_unspendable)
  File "/electrumx/electrumx/server/block_processor.py", line 547, in backup_txs
    assert n == 0
AssertionError