"""
Ported to Python 3.
"""

from future.utils import bchr

# system-level upload+download roundtrip test, but using shares created from
# a previous run. This asserts that the current code is capable of decoding
# shares from a previous version.

from typing import Any

import six
import os
from twisted.trial import unittest
from twisted.internet import defer, reactor
from allmydata import uri
from allmydata.storage.server import storage_index_to_dir
from allmydata.util import base32, fileutil, spans, log, hashutil
from allmydata.util.consumer import download_to_data, MemoryConsumer
from allmydata.immutable import upload, layout
from allmydata.test.no_network import GridTestMixin, NoNetworkServer
from allmydata.test.common import ShouldFailMixin
from allmydata.interfaces import NotEnoughSharesError, NoSharesError, \
     DownloadStopped
from allmydata.immutable.downloader.common import BadSegmentNumberError, \
     BadCiphertextHashError, COMPLETE, OVERDUE, DEAD
from allmydata.immutable.downloader.status import DownloadStatus
from allmydata.immutable.downloader.fetcher import SegmentFetcher
from allmydata.codec import CRSDecoder
from foolscap.eventual import eventually, fireEventually, flushEventualQueue

if six.PY3:
    long = int

plaintext = b"This is a moderate-sized file.\n" * 10
mutable_plaintext = b"This is a moderate-sized mutable file.\n" * 10

# this chunk was generated by create_share(), written to disk, then pasted
# into this file. These shares were created by 1.2.0-r3247, a version that's
# probably fairly close to 1.3.0 .
#--------- BEGIN stored_shares.py --------------
immutable_uri = b"URI:CHK:g4i6qkk7mlj4vkl5ncg6dwo73i:qcas2ebousfk3q5rkl2ncayeku52kpyse76v5yeel2t2eaa4f6ha:3:10:310"
immutable_shares = {
 0: { # client[0]
  0: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazmksehmgmlmmeqkbxbljh5qnfq36b7h5ukgqccmy3665khphcxihkce7jukeuegdxtn26p353ork6qihitbshwucpopzvdnpkflg6vbvko7ohcmxjywpdkvjmuzq6hysxfl74mamn224nrsyl7czmvtwtss6kkzljridkffeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7y5y3kjcfritduzdk5rvwqs4lwzvb7fgvljgozbbtamhoriuzaeruaaqt2fbbxr5yv4vqeabkjqow6sd73dfqab3qban3htx6rn2y6mujdwaacbpvbyim4ewanv2vku44tunk7vdjkty2wkfm3jg67pqmm2newyib4aafazigyt6kxmirnlio5sdvbkvh43rwpctm6coigl64chn6z7w45rcaaccvmfgplu4kz5erphnx3xhzclypawi2j5zsvewmn4s2wbba4k2ktaab45y3kjcfritduzdk5rvwqs4lwzvb7fgvljgozbbtamhoriuzaeruaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabj5uiln36za2n4oyier7k5e4sx6newmmflfqhj7xffy32p5iohlyf33bdx5dafkfwr7rxwxjcsg3ljflkaae537llwnnykgf36h52dojfplbwi"),
  5: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazmsdsvwbnfx2rnh7dusqniqomsdeetuafps6cawyb4pzxpkzal7w5ufaknxfnqw2qywv4c3a2zlumb2x2rx5osbxd3kqmebjndqf7zihbtagqczgwrka5rnywtsaeyijyh26okua2u7loep2nzo5etirjrxmp3yxpb4pheusaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7zs3zcg7igd2xoa4eu3lffqginpmoxrshqe6n3hzpocihgeu4vvymaadjz54nelgyi47767pkbsjwdjgsv7uyd5ntrztw6juavj7sd7wx7aaacx7wxlycyjniwxvby4ar546ncb4d3jnbhssnq4n4l4xeajurmn5diabgxwi6i5d2ysny3vavrm3a5lsuvng5mhbzk7axesyeddzw6uzmnluaakglpei35aypk5ydqstnmuwazbv5r26gi6atzxm7f5yja4ystswxbqaakbsqnrh4voyrc2wq53ehkcvkpzxdm6fgz4e4qmx5yeo35t7nz3ceaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabj5uiln36za2n4oyier7k5e4sx6newmmflfqhj7xffy32p5iohlyf33bdx5dafkfwr7rxwxjcsg3ljflkaae537llwnnykgf36h52dojfplbwi"),
    },
 1: { # client[1]
  2: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazmj7um4zfgqo35m62ln6has6xz43klzjphj5eg46mb5x2jzgr6x6zb4voveo5uef53xbjbktr5rlupomy7x5b34amqeeg4r6obt6kpo2x4s3m3cwoo54oijyqfms3n3fethykhtglc47r4ci7ugqgz5d5fap3xzyhm4ehaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7zqkzg32wa74epeppqwneujs6tjptlm4qw75hoafobsoif3ok5odkaarmcwjw6vqh7bdzd34ftjfcmxu2l423hefx7j3qblqmtsbo3sxlq2qaewyffwgzojfi4uj2praj5azehnr4fhan5kdyewhtfncrqzoe42ijeaaikvquz5otrlhusf45w7o47ejpb4czdjhxgkuszrxslkyeedrljkmaabigkbwe7sv3celk2dxmq5ikvj7g4ntyu3hqtsbs7xar3pwp5xhmiqaa6k7uub7uqlamlqi2oduautemch242scu7cfor6kedxs6mm3uwjsmaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabdp37hh2k4ys4d7qusb5e3dakjntythtcwcwfok7e52pu64zn4wrwbtlkzxzntwuwemi6e6mek5n4i7h3bw7nkat2zmqieftinxgzl2jfplbwi"),
  7: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaaznhsh2frhzxbutelvddtbuf3tfilhcj2zi3cxjyzy7pg7ewamazcblv76mvey54fxmch64chqfi24jmondc4uzitby3wjeui4nfp7kv6ufo67exptkvwk7cnbouvjiapyqzrps4r6ise4jhlr7mtp2tlizb5hyaqm3fhsvrmqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl72g6h2oewtfcwgupjbjnh4k5k6d3k2fpi2q6nyidh3yo5ui6cslreaajms7f3pcsywhjbgrybzp64jzlsyjqbu7h4hvdlwf77ar6l63imdeqaaudfa3cpzk5rcfvnb3wioufku7togz4kntyjzazp3qi5x3h63tweiaagtt3y2iwnqrz77566udetmgsnfl7jqh23hdthn4tibkt7eh7np6aaakvpbzjdki64qaigkdj2bven3uigxbpurpwtrkjs4b6habv2ls7zqaac2g6h2oewtfcwgupjbjnh4k5k6d3k2fpi2q6nyidh3yo5ui6cslreaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabdp37hh2k4ys4d7qusb5e3dakjntythtcwcwfok7e52pu64zn4wrwbtlkzxzntwuwemi6e6mek5n4i7h3bw7nkat2zmqieftinxgzl2jfplbwi"),
    },
 2: { # client[2]
  1: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazmkrwrt6figauxkgqyk3nggp5eeoeq5htt7tke4gfqj2u5roieslao4fldcwlq4btzk4brhkaerqiih6mhudotttrb6xzmvnqgg33fjcqeuw6teb3gml2pmhsezisa5svnzlvqnbaz6kzdmhisbwgu6ocexf2ge2rvc67gneqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl72piueg6hxcxswaqafjgb232ip7mmwaahoaebxm6o72fxldzsreoyaaif6uhbbtqsybwxkvkttsorvl6unfkpdkzivtne3356brtjus3bahqaee6riin4pofpfmbaaksmdvxuq76yzmaao4aidoz457ulowhtfci5qaafazigyt6kxmirnlio5sdvbkvh43rwpctm6coigl64chn6z7w45rcaaccvmfgplu4kz5erphnx3xhzclypawi2j5zsvewmn4s2wbba4k2ktaab45y3kjcfritduzdk5rvwqs4lwzvb7fgvljgozbbtamhoriuzaeruaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaugotrr7enihxy2b2nwodhxabihaf3ewc2hmcdjsqx5hi4h3rn7gnvpt3lzzo5qgbnlp4dybwr7dn7vu5hsiyo5pedlqcasb7csiuojfplbwi"),
  6: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazm34cgyp37ou5ohrofmk6bf5gcppxeb2njwmiwasn3uh4ykeocvq4vydsw36ksh63fcil3o257zupffrruiuqlwjvbdcdjiuqrojiromunzxxc34io7zlfafprzlvmztph4qsp67ozxmwvivqwtvu6ckr7pffsikgi2supviqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7zlyoki2shxeacbsq2oqnjdo5cbvyl5el5u4ksmxapryanos4x6maaajms7f3pcsywhjbgrybzp64jzlsyjqbu7h4hvdlwf77ar6l63imdeqaaudfa3cpzk5rcfvnb3wioufku7togz4kntyjzazp3qi5x3h63tweiaagtt3y2iwnqrz77566udetmgsnfl7jqh23hdthn4tibkt7eh7np6aaakvpbzjdki64qaigkdj2bven3uigxbpurpwtrkjs4b6habv2ls7zqaac2g6h2oewtfcwgupjbjnh4k5k6d3k2fpi2q6nyidh3yo5ui6cslreaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaugotrr7enihxy2b2nwodhxabihaf3ewc2hmcdjsqx5hi4h3rn7gnvpt3lzzo5qgbnlp4dybwr7dn7vu5hsiyo5pedlqcasb7csiuojfplbwi"),
    },
 3: { # client[3]
  4: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaaznjqn7ehmj6f4p3fjyliuvwnfothumsfhs7ienw4uln6joaxopqlmcy5daa4njrkgj7nqm6tpnmz2dci2b356pljv4zjj5ayzfihi4g26qdei7kjtegjuv4d3k3t4orpufnft3edbondkpj5etjczwhyulukzuy5socyivdfqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7zpmr4r2hvre3rxkblczwb2xfjk2n2yodsv6bojfqightn5jsy2xiaatl3epeor5mjg4n2qkywnqovzkkwtowdq4vpqlsjmcbr43pkmwgv2aacx7wxlycyjniwxvby4ar546ncb4d3jnbhssnq4n4l4xeajurmn5diaagtt3y2iwnqrz77566udetmgsnfl7jqh23hdthn4tibkt7eh7np6aaakglpei35aypk5ydqstnmuwazbv5r26gi6atzxm7f5yja4ystswxbqaakbsqnrh4voyrc2wq53ehkcvkpzxdm6fgz4e4qmx5yeo35t7nz3ceaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacifoqlus3puiqkekp6g6fdecjcx2bak27angodamzoxugovlhtcj5xbly7teqwmf73fqk3clyfvs6hdauq5qnqahlxlmp2vrmnneedgjfplbwi"),
  9: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazn2tz3qt62bgsdnvksvdegsylb2kbltouheryflpho7hugme7svk7so2v7hmcgc43tcyugybuqzgifvkllikfiiezvml7ilolb7ivwvrv4d4t2gbywa44ibqwogmjtffta4b2sfwqebfg7pptergeqm5wo3tndtf7p3vftabqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl7y3m26swfhsb66ze4cmyhohaksid7fyljgkhag32ibc7vx2yj4j5saayg3gxuvrj4qpxwjhatgb3rycusa7zoc2jsrybw6saix5n6wcpcpmqaamxjsc6bwv4w4or2oylltmsbfbobvmenj3sa6lnq6iy4tugsnv72eaaybvqu3gmlomi3dnf2tum3hoseavpesyia2i2wqgwbmbtrgmotu6oaadirzs2idl54toffh4a2hehvg2e3zoed4dr6pcdpuqpnz2knte7gqqac6kfatp33ianoqvg6mdd4vaxa27lo6vpugbcvanhskaqq2kewn6kwaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacifoqlus3puiqkekp6g6fdecjcx2bak27angodamzoxugovlhtcj5xbly7teqwmf73fqk3clyfvs6hdauq5qnqahlxlmp2vrmnneedgjfplbwi"),
    },
 4: { # client[4]
  3: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaazmbduh5nwvcvpgrihhnjxacz2jvzu3prrdqewo3vmxkhu5yd3fa3eil56fyh5l7ojimghwbf2o6ri7cmppr34qflr5o4w6s5fekxhdt3qvlgsw5yp5wrmjjffhph5czd5kzoo7yyg5x3wgxxzdvwtuom2c5olao62ep77b7wqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl73mcs3dmxesuoke5hyqe6qmsdwy6ctqg6vb4cldzswriymxconeesaarmcwjw6vqh7bdzd34ftjfcmxu2l423hefx7j3qblqmtsbo3sxlq2qaaudfa3cpzk5rcfvnb3wioufku7togz4kntyjzazp3qi5x3h63tweiaaikvquz5otrlhusf45w7o47ejpb4czdjhxgkuszrxslkyeedrljkmaajnqklmns4skrzitu7cat2bsio3dykoa32uhqjmpgk2fdbs4jzuqsiaa6k7uub7uqlamlqi2oduautemch242scu7cfor6kedxs6mm3uwjsmaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadusyxmwhtnfdeewwgxd25fwixycfdcy46ifqv4dhga23fko6dbl4ywo2d27n3zh3wd6zumhupvmtgspqrh6t7wbsghruzqd3imbo2tojfplbwi"),
  8: base32.a2b(b"aaaaaaiaaacyeaaaaaaqaaaaaeaaaadiaaaaa2aaaaaciaaaacgaaaaavqaaaagmaaaab3aaaaaznjzqcxwyhgwlcpzvfb2berhoyw47h72gkzofwgksryqd4r6xlyougvyg4p3wkz7u37zllskeswuuh4w2rylbxecomnmqfv7n5ex3thjzq7ykr7gjkvq3kmrlhmxu3wnsr4ipsdn546btavjzc6yppoii2mxgnnk4qbxqrltaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaactwxn7tsxj2nh6skwbghycguqfj7xrpegeporex5ebctynbgbl72kfatp33ianoqvg6mdd4vaxa27lo6vpugbcvanhskaqq2kewn6kwaayg3gxuvrj4qpxwjhatgb3rycusa7zoc2jsrybw6saix5n6wcpcpmqaamxjsc6bwv4w4or2oylltmsbfbobvmenj3sa6lnq6iy4tugsnv72eaaybvqu3gmlomi3dnf2tum3hoseavpesyia2i2wqgwbmbtrgmotu6oaadirzs2idl54toffh4a2hehvg2e3zoed4dr6pcdpuqpnz2knte7gqqac6kfatp33ianoqvg6mdd4vaxa27lo6vpugbcvanhskaqq2kewn6kwaaaae4gg33emvrv63tbnvstumz2mnzhglddn5sgky27obqxeylnom5dqortgezc2mzngeycyy3spfyhi5dfpb2f62dbonudumzshkl7fjw5sp7x3yw4sdhze6qf7zgsjocpqtwl2gj5o6vufvixto3u2lddoj4xa5dumv4hix3sn5xxix3imfzwqortgi5fhno37hfotu2p5evmcmpqenjakt7pc6imi65cjp2icfhq2cmcx7rmnzswkzdfmrpxg2dbojsxgorrhizsy3tvnvpxgzlhnvsw45dthiytumjmonswo3lfnz2f643jpjstumz2gmyteldtnbqxezk7ojxw65c7nbqxg2b2gmzdubzmius26hljzu4j7gq5hdshwueqcfjc2bmveiyqbdxgyejetzovfrzws6tfhiztumzrgawhiyljnrpwg33emvrv64dbojqw24z2ha5dgmjsfuzs2mjqfr2g65dbnrpxg2dbojsxgorshiytalaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadusyxmwhtnfdeewwgxd25fwixycfdcy46ifqv4dhga23fko6dbl4ywo2d27n3zh3wd6zumhupvmtgspqrh6t7wbsghruzqd3imbo2tojfplbwi"),
    },
}

mutable_uri = b"URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
mutable_shares = {
 0: { # client[0]
  2: base32.a2b(b" "),
  7: base32.a2b(b" "),
    },
 1: { # client[1]
  3: base32.a2b(b" "),
  8: base32.a2b(b" "),
    },
 2: { # client[2]
  4: base32.a2b(b" "),
  9: base32.a2b(b" "),
    },
 3: { # client[3]
  1: base32.a2b(b" "),
  6: base32.a2b(b" "),
    },
 4: { # client[4]
  0: base32.a2b(b" "),
  5: base32.a2b(b" "),
    },
}
#--------- END stored_shares.py ----------------

class _Base(GridTestMixin, ShouldFailMixin):

    def create_shares(self, ignored=None):
        u = upload.Data(plaintext, None)
        d = self.c0.upload(u)
        f = open("stored_shares.py", "w")
        def _created_immutable(ur):
            # write the generated shares and URI to a file, which can then be
            # incorporated into this one next time.
            f.write('immutable_uri = b"%s"\n' % ur.get_uri())
            f.write('immutable_shares = {\n')
            si = uri.from_string(ur.get_uri()).get_storage_index()
            si_dir = storage_index_to_dir(si)
            for (i,ss,ssdir) in self.iterate_servers():
                sharedir = os.path.join(ssdir, "shares", si_dir)
                shares = {}
                for fn in os.listdir(sharedir):
                    shnum = int(fn)
                    sharedata = open(os.path.join(sharedir, fn), "rb").read()
                    shares[shnum] = sharedata
                fileutil.rm_dir(sharedir)
                if shares:
                    f.write(' %d: { # client[%d]\n' % (i, i))
                    for shnum in sorted(shares.keys()):
                        f.write('  %d: base32.a2b(b"%s"),\n' %
                                (shnum, base32.b2a(shares[shnum])))
                    f.write('    },\n')
            f.write('}\n')
            f.write('\n')

        d.addCallback(_created_immutable)

        d.addCallback(lambda ignored:
                      self.c0.create_mutable_file(mutable_plaintext))
        def _created_mutable(n):
            f.write('mutable_uri = b"%s"\n' % n.get_uri())
            f.write('mutable_shares = {\n')
            si = uri.from_string(n.get_uri()).get_storage_index()
            si_dir = storage_index_to_dir(si)
            for (i,ss,ssdir) in self.iterate_servers():
                sharedir = os.path.join(ssdir, "shares", si_dir)
                shares = {}
                for fn in os.listdir(sharedir):
                    shnum = int(fn)
                    sharedata = open(os.path.join(sharedir, fn), "rb").read()
                    shares[shnum] = sharedata
                fileutil.rm_dir(sharedir)
                if shares:
                    f.write(' %d: { # client[%d]\n' % (i, i))
                    for shnum in sorted(shares.keys()):
                        f.write('  %d: base32.a2b(b"%s"),\n' %
                                (shnum, base32.b2a(shares[shnum])))
                    f.write('    },\n')
            f.write('}\n')

            f.close()
        d.addCallback(_created_mutable)

        def _done(ignored):
            f.close()
        d.addCallback(_done)

        return d

    def load_shares(self, ignored=None):
        # this uses the data generated by create_shares() to populate the
        # storage servers with pre-generated shares
        si = uri.from_string(immutable_uri).get_storage_index()
        si_dir = storage_index_to_dir(si)
        for i in immutable_shares:
            shares = immutable_shares[i]
            for shnum in shares:
                dn = os.path.join(self.get_serverdir(i), "shares", si_dir)
                fileutil.make_dirs(dn)
                fn = os.path.join(dn, str(shnum))
                f = open(fn, "wb")
                f.write(shares[shnum])
                f.close()

        si = uri.from_string(mutable_uri).get_storage_index()
        si_dir = storage_index_to_dir(si)
        for i in mutable_shares:
            shares = mutable_shares[i]
            for shnum in shares:
                dn = os.path.join(self.get_serverdir(i), "shares", si_dir)
                fileutil.make_dirs(dn)
                fn = os.path.join(dn, str(shnum))
                f = open(fn, "wb")
                f.write(shares[shnum])
                f.close()

    def download_immutable(self, ignored=None):
        n = self.c0.create_node_from_uri(immutable_uri)
        d = download_to_data(n)
        def _got_data(data):
            self.failUnlessEqual(data, plaintext)
        d.addCallback(_got_data)
        # make sure we can use the same node twice
        d.addCallback(lambda ign: download_to_data(n))
        d.addCallback(_got_data)
        return d

    def download_mutable(self, ignored=None):
        n = self.c0.create_node_from_uri(mutable_uri)
        d = n.download_best_version()
        def _got_data(data):
            self.failUnlessEqual(data, mutable_plaintext)
        d.addCallback(_got_data)
        return d

class DownloadTest(_Base, unittest.TestCase):
    def test_download(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # do this to create the shares
        #return self.create_shares()

        self.load_shares()
        d = self.download_immutable()
        d.addCallback(self.download_mutable)
        return d

    def test_download_failover(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        self.load_shares()
        si = uri.from_string(immutable_uri).get_storage_index()
        si_dir = storage_index_to_dir(si)

        n = self.c0.create_node_from_uri(immutable_uri)
        d = download_to_data(n)
        def _got_data(data):
            self.failUnlessEqual(data, plaintext)
        d.addCallback(_got_data)

        def _clobber_some_shares(ign):
            # find the three shares that were used, and delete them. Then
            # download again, forcing the downloader to fail over to other
            # shares
            for s in n._cnode._node._shares:
                for clientnum in immutable_shares:
                    for shnum in immutable_shares[clientnum]:
                        if s._shnum == shnum:
                            fn = os.path.join(self.get_serverdir(clientnum),
                                              "shares", si_dir, str(shnum))
                            os.unlink(fn)
        d.addCallback(_clobber_some_shares)
        d.addCallback(lambda ign: download_to_data(n))
        d.addCallback(_got_data)

        def _clobber_most_shares(ign):
            # delete all but one of the shares that are still alive
            live_shares = [s for s in n._cnode._node._shares if s.is_alive()]
            save_me = live_shares[0]._shnum
            for clientnum in immutable_shares:
                for shnum in immutable_shares[clientnum]:
                    if shnum == save_me:
                        continue
                    fn = os.path.join(self.get_serverdir(clientnum),
                                      "shares", si_dir, str(shnum))
                    if os.path.exists(fn):
                        os.unlink(fn)
            # now the download should fail with NotEnoughSharesError
            return self.shouldFail(NotEnoughSharesError, "1shares", None,
                                   download_to_data, n)
        d.addCallback(_clobber_most_shares)

        def _clobber_all_shares(ign):
            # delete the last remaining share
            for clientnum in immutable_shares:
                for shnum in immutable_shares[clientnum]:
                    fn = os.path.join(self.get_serverdir(clientnum),
                                      "shares", si_dir, str(shnum))
                    if os.path.exists(fn):
                        os.unlink(fn)
            # now a new download should fail with NoSharesError. We want a
            # new ImmutableFileNode so it will forget about the old shares.
            # If we merely called create_node_from_uri() without first
            # dereferencing the original node, the NodeMaker's _node_cache
            # would give us back the old one.
            n = None
            n = self.c0.create_node_from_uri(immutable_uri)
            return self.shouldFail(NoSharesError, "0shares", None,
                                   download_to_data, n)
        d.addCallback(_clobber_all_shares)
        return d

    def test_lost_servers(self):
        # while downloading a file (after seg[0], before seg[1]), lose the
        # three servers that we were using. The download should switch over
        # to other servers.
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, so we can catch the download
        # in the middle.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs
        d = self.c0.upload(u)
        def _uploaded(ur):
            self.uri = ur.get_uri()
            self.n = self.c0.create_node_from_uri(self.uri)
            return download_to_data(self.n)
        d.addCallback(_uploaded)
        def _got_data(data):
            self.failUnlessEqual(data, plaintext)
        d.addCallback(_got_data)
        def _kill_some_shares():
            # find the shares that were used and delete them
            shares = self.n._cnode._node._shares
            self.killed_share_nums = sorted([s._shnum for s in shares])

            # break the RIBucketReader references
            # (we don't break the RIStorageServer references, because that
            # isn't needed to test the current downloader implementation)
            for s in shares:
                s._rref.broken = True
        def _download_again(ign):
            # download again, deleting some shares after the first write
            # to the consumer
            c = StallingConsumer(_kill_some_shares)
            return self.n.read(c)
        d.addCallback(_download_again)
        def _check_failover(c):
            self.failUnlessEqual(b"".join(c.chunks), plaintext)
            shares = self.n._cnode._node._shares
            shnums = sorted([s._shnum for s in shares])
            self.failIfEqual(shnums, self.killed_share_nums)
        d.addCallback(_check_failover)
        return d

    def test_long_offset(self):
        # bug #1154: mplayer doing a seek-to-end results in an offset of type
        # 'long', rather than 'int', and apparently __len__ is required to
        # return an int. Rewrote Spans/DataSpans to provide s.len() instead
        # of len(s) .
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)

        c = MemoryConsumer()
        d = n.read(c, int(0), int(10))
        d.addCallback(lambda c: len(b"".join(c.chunks)))
        d.addCallback(lambda size: self.failUnlessEqual(size, 10))
        return d

    def test_badguess(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)

        # Cause the downloader to guess a segsize that's too low, so it will
        # ask for a segment number that's too high (beyond the end of the
        # real list, causing BadSegmentNumberError), to exercise
        # Segmentation._retry_bad_segment
        n._cnode._maybe_create_download_node()
        n._cnode._node._build_guessed_tables(90)

        con1 = MemoryConsumer()
        # plaintext size of 310 bytes, wrong-segsize of 90 bytes, will make
        # us think that file[180:200] is in the third segment (segnum=2), but
        # really there's only one segment
        d = n.read(con1, 180, 20)
        def _done(res):
            self.failUnlessEqual(b"".join(con1.chunks), plaintext[180:200])
        d.addCallback(_done)
        return d

    def test_simultaneous_badguess(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, and a non-default segsize, to
        # exercise the offset-guessing code. Because we don't tell the
        # downloader about the unusual segsize, it will guess wrong, and have
        # to do extra roundtrips to get the correct data.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs, 8-wide hashtree
        con1 = MemoryConsumer()
        con2 = MemoryConsumer()
        d = self.c0.upload(u)
        def _uploaded(ur):
            n = self.c0.create_node_from_uri(ur.get_uri())
            d1 = n.read(con1, 70, 20)
            d2 = n.read(con2, 140, 20)
            return defer.gatherResults([d1,d2])
        d.addCallback(_uploaded)
        def _done(res):
            self.failUnlessEqual(b"".join(con1.chunks), plaintext[70:90])
            self.failUnlessEqual(b"".join(con2.chunks), plaintext[140:160])
        d.addCallback(_done)
        return d

    def test_simultaneous_goodguess(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, and a non-default segsize, to
        # exercise the offset-guessing code. This time we *do* tell the
        # downloader about the unusual segsize, so it can guess right.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs, 8-wide hashtree
        con1 = MemoryConsumer()
        con2 = MemoryConsumer()
        d = self.c0.upload(u)
        def _uploaded(ur):
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)
            d1 = n.read(con1, 70, 20)
            d2 = n.read(con2, 140, 20)
            return defer.gatherResults([d1,d2])
        d.addCallback(_uploaded)
        def _done(res):
            self.failUnlessEqual(b"".join(con1.chunks), plaintext[70:90])
            self.failUnlessEqual(b"".join(con2.chunks), plaintext[140:160])
        d.addCallback(_done)
        return d

    def test_sequential_goodguess(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        data = (plaintext*100)[:30000] # multiple of k

        # upload a file with multiple segments, and a non-default segsize, to
        # exercise the offset-guessing code. This time we *do* tell the
        # downloader about the unusual segsize, so it can guess right.
        u = upload.Data(data, None)
        u.max_segment_size = 6000 # 5 segs, 8-wide hashtree
        con1 = MemoryConsumer()
        con2 = MemoryConsumer()
        d = self.c0.upload(u)
        def _uploaded(ur):
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)
            d = n.read(con1, 12000, 20)
            def _read1(ign):
                self.failUnlessEqual(b"".join(con1.chunks), data[12000:12020])
                return n.read(con2, 24000, 20)
            d.addCallback(_read1)
            def _read2(ign):
                self.failUnlessEqual(b"".join(con2.chunks), data[24000:24020])
            d.addCallback(_read2)
            return d
        d.addCallback(_uploaded)
        return d


    def test_simultaneous_get_blocks(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        self.load_shares()
        stay_empty = []

        n = self.c0.create_node_from_uri(immutable_uri)
        d = download_to_data(n)
        def _use_shares(ign):
            shares = list(n._cnode._node._shares)
            s0 = shares[0]
            # make sure .cancel works too
            o0 = s0.get_block(0)
            o0.subscribe(lambda **kwargs: stay_empty.append(kwargs))
            o1 = s0.get_block(0)
            o2 = s0.get_block(0)
            o0.cancel()
            o3 = s0.get_block(1) # state=BADSEGNUM
            d1 = defer.Deferred()
            d2 = defer.Deferred()
            d3 = defer.Deferred()
            o1.subscribe(lambda **kwargs: d1.callback(kwargs))
            o2.subscribe(lambda **kwargs: d2.callback(kwargs))
            o3.subscribe(lambda **kwargs: d3.callback(kwargs))
            return defer.gatherResults([d1,d2,d3])
        d.addCallback(_use_shares)
        def _done(res):
            r1,r2,r3 = res
            self.failUnlessEqual(r1["state"], "COMPLETE")
            self.failUnlessEqual(r2["state"], "COMPLETE")
            self.failUnlessEqual(r3["state"], "BADSEGNUM")
            self.failUnless("block" in r1)
            self.failUnless("block" in r2)
            self.failIf(stay_empty)
        d.addCallback(_done)
        return d

    def test_simul_1fail_1cancel(self):
        # This exercises an mplayer behavior in ticket #1154. I believe that
        # mplayer made two simultaneous webapi GET requests: first one for an
        # index region at the end of the (mp3/video) file, then one for the
        # first block of the file (the order doesn't really matter). All GETs
        # failed (NoSharesError) because of the type(__len__)==long bug. Each
        # GET submitted a DownloadNode.get_segment() request, which was
        # queued by the DN (DN._segment_requests), so the second one was
        # blocked waiting on the first one. When the first one failed,
        # DN.fetch_failed() was invoked, which errbacks the first GET, but
        # left the other one hanging (the lost-progress bug mentioned in
        # #1154 comment 10)
        #
        # Then mplayer sees that the index region GET failed, so it cancels
        # the first-block GET (by closing the HTTP request), triggering
        # stopProducer. The second GET was waiting in the Deferred (between
        # n.get_segment() and self._request_retired), so its
        # _cancel_segment_request was active, so was invoked. However,
        # DN._active_segment was None since it was not working on any segment
        # at that time, hence the error in #1154.

        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, so we can catch the download
        # in the middle. Tell the downloader, so it can guess correctly.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs
        d = self.c0.upload(u)
        def _uploaded(ur):
            # corrupt all the shares so the download will fail
            def _corruptor(s, debug=False):
                which = 48 # first byte of block0
                return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:]
            self.corrupt_all_shares(ur.get_uri(), _corruptor)
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)
            con1 = MemoryConsumer()
            con2 = MemoryConsumer()
            d = n.read(con1, int(0), int(20))
            d2 = n.read(con2, int(140), int(20))
            # con2 will be cancelled, so d2 should fail with DownloadStopped
            def _con2_should_not_succeed(res):
                self.fail("the second read should not have succeeded")
            def _con2_failed(f):
                self.failUnless(f.check(DownloadStopped))
            d2.addCallbacks(_con2_should_not_succeed, _con2_failed)

            def _con1_should_not_succeed(res):
                self.fail("the first read should not have succeeded")
            def _con1_failed(f):
                self.failUnless(f.check(NoSharesError))
                con2.producer.stopProducing()
                return d2
            d.addCallbacks(_con1_should_not_succeed, _con1_failed)
            return d
        d.addCallback(_uploaded)
        return d

    def test_simultaneous_onefails(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, so we can catch the download
        # in the middle. Tell the downloader, so it can guess correctly.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs
        d = self.c0.upload(u)
        def _uploaded(ur):
            # corrupt all the shares so the download will fail
            def _corruptor(s, debug=False):
                which = 48 # first byte of block0
                return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:]
            self.corrupt_all_shares(ur.get_uri(), _corruptor)
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)
            con1 = MemoryConsumer()
            con2 = MemoryConsumer()
            d = n.read(con1, int(0), int(20))
            d2 = n.read(con2, int(140), int(20))
            # con2 should wait for con1 to fail and then con2 should succeed.
            # In particular, we should not lose progress. If this test fails,
            # it will fail with a timeout error.
            def _con2_should_succeed(res):
                # this should succeed because we only corrupted the first
                # segment of each share. The segment that holds [140:160] is
                # fine, as are the hash chains and UEB.
                self.failUnlessEqual(b"".join(con2.chunks), plaintext[140:160])
            d2.addCallback(_con2_should_succeed)

            def _con1_should_not_succeed(res):
                self.fail("the first read should not have succeeded")
            def _con1_failed(f):
                self.failUnless(f.check(NoSharesError))
                # we *don't* cancel the second one here: this exercises a
                # lost-progress bug from #1154. We just wait for it to
                # succeed.
                return d2
            d.addCallbacks(_con1_should_not_succeed, _con1_failed)
            return d
        d.addCallback(_uploaded)
        return d

    def test_download_no_overrun(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        self.load_shares()

        # tweak the client's copies of server-version data, so it believes
        # that they're old and can't handle reads that overrun the length of
        # the share. This exercises a different code path.
        for s in self.c0.storage_broker.get_connected_servers():
            v = s.get_version()
            v1 = v[b"http://allmydata.org/tahoe/protocols/storage/v1"]
            v1[b"tolerates-immutable-read-overrun"] = False

        n = self.c0.create_node_from_uri(immutable_uri)
        d = download_to_data(n)
        def _got_data(data):
            self.failUnlessEqual(data, plaintext)
        d.addCallback(_got_data)
        return d

    def test_download_segment(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        cn = n._cnode
        (d,c) = cn.get_segment(0)
        def _got_segment(offset_and_data_and_decodetime):
            (offset, data, decodetime) = offset_and_data_and_decodetime
            self.failUnlessEqual(offset, 0)
            self.failUnlessEqual(len(data), len(plaintext))
        d.addCallback(_got_segment)
        return d

    def test_download_segment_cancel(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        cn = n._cnode
        (d,c) = cn.get_segment(0)
        fired = []
        d.addCallback(fired.append)
        c.cancel()
        d = fireEventually()
        d.addCallback(flushEventualQueue)
        def _check(ign):
            self.failUnlessEqual(fired, [])
        d.addCallback(_check)
        return d

    def test_download_bad_segment(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        cn = n._cnode
        def _try_download():
            (d,c) = cn.get_segment(1)
            return d
        d = self.shouldFail(BadSegmentNumberError, "badseg",
                            "segnum=1, numsegs=1",
                            _try_download)
        return d

    def test_download_segment_terminate(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        cn = n._cnode
        (d,c) = cn.get_segment(0)
        fired = []
        d.addCallback(fired.append)
        self.c0.terminator.disownServiceParent()
        d = fireEventually()
        d.addCallback(flushEventualQueue)
        def _check(ign):
            self.failUnlessEqual(fired, [])
        d.addCallback(_check)
        return d

    def test_pause(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        c = PausingConsumer()
        d = n.read(c)
        def _downloaded(mc):
            newdata = b"".join(mc.chunks)
            self.failUnlessEqual(newdata, plaintext)
        d.addCallback(_downloaded)
        return d

    def test_pause_then_stop(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        c = PausingAndStoppingConsumer()
        d = self.shouldFail(DownloadStopped, "test_pause_then_stop",
                            "our Consumer called stopProducing()",
                            n.read, c)
        return d

    def test_stop(self):
        # use a download target that stops after the first segment (#473)
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)
        c = StoppingConsumer()
        d = self.shouldFail(DownloadStopped, "test_stop",
                            "our Consumer called stopProducing()",
                            n.read, c)
        return d

    def test_stop_immediately(self):
        # and a target that stops right after registerProducer (maybe #1154)
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)

        c = ImmediatelyStoppingConsumer() # stops after registerProducer
        d = self.shouldFail(DownloadStopped, "test_stop_immediately",
                            "our Consumer called stopProducing()",
                            n.read, c)
        return d

    def test_stop_immediately2(self):
        # and a target that stops right after registerProducer (maybe #1154)
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()
        n = self.c0.create_node_from_uri(immutable_uri)

        c = MemoryConsumer()
        d0 = n.read(c)
        c.producer.stopProducing()
        d = self.shouldFail(DownloadStopped, "test_stop_immediately",
                            "our Consumer called stopProducing()",
                            lambda: d0)
        return d

    def test_download_segment_bad_ciphertext_hash(self):
        # The crypttext_hash_tree asserts the integrity of the decoded
        # ciphertext, and exists to detect two sorts of problems. The first
        # is a bug in zfec decode. The second is the "two-sided t-shirt"
        # attack (found by Christian Grothoff), in which a malicious uploader
        # creates two sets of shares (one for file A, second for file B),
        # uploads a combination of them (shares 0-4 of A, 5-9 of B), and then
        # builds an otherwise normal UEB around those shares: their goal is
        # to give their victim a filecap which sometimes downloads the good A
        # contents, and sometimes the bad B contents, depending upon which
        # servers/shares they can get to. Having a hash of the ciphertext
        # forces them to commit to exactly one version. (Christian's prize
        # for finding this problem was a t-shirt with two sides: the shares
        # of file A on the front, B on the back).

        # creating a set of shares with this property is too hard, although
        # it'd be nice to do so and confirm our fix. (it requires a lot of
        # tampering with the uploader). So instead, we just damage the
        # decoder. The tail decoder is rebuilt each time, so we need to use a
        # file with multiple segments.
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        u = upload.Data(plaintext, None)
        u.max_segment_size = 60 # 6 segs
        d = self.c0.upload(u)
        def _uploaded(ur):
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)

            d = download_to_data(n)
            def _break_codec(data):
                # the codec isn't created until the UEB is retrieved
                node = n._cnode._node
                vcap = node._verifycap
                k, N = vcap.needed_shares, vcap.total_shares
                bad_codec = BrokenDecoder()
                bad_codec.set_params(node.segment_size, k, N)
                node._codec = bad_codec
            d.addCallback(_break_codec)
            # now try to download it again. The broken codec will provide
            # ciphertext that fails the hash test.
            d.addCallback(lambda ign:
                          self.shouldFail(BadCiphertextHashError, "badhash",
                                          "hash failure in "
                                          "ciphertext_hash_tree: segnum=0",
                                          download_to_data, n))
            return d
        d.addCallback(_uploaded)
        return d

    def OFFtest_download_segment_XXX(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file with multiple segments, and a non-default segsize, to
        # exercise the offset-guessing code. This time we *do* tell the
        # downloader about the unusual segsize, so it can guess right.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 70 # 5 segs, 8-wide hashtree
        con1 = MemoryConsumer()
        con2 = MemoryConsumer()
        d = self.c0.upload(u)
        def _uploaded(ur):
            n = self.c0.create_node_from_uri(ur.get_uri())
            n._cnode._maybe_create_download_node()
            n._cnode._node._build_guessed_tables(u.max_segment_size)
            d1 = n.read(con1, 70, 20)
            #d2 = n.read(con2, 140, 20)
            d2 = defer.succeed(None)
            return defer.gatherResults([d1,d2])
        d.addCallback(_uploaded)
        def _done(res):
            self.failUnlessEqual(b"".join(con1.chunks), plaintext[70:90])
            self.failUnlessEqual(b"".join(con2.chunks), plaintext[140:160])
        #d.addCallback(_done)
        return d

    def test_duplicate_shares(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        self.load_shares()
        # make sure everybody has a copy of sh0. The second server contacted
        # will report two shares, and the ShareFinder will handle the
        # duplicate by attaching both to the same CommonShare instance.
        si = uri.from_string(immutable_uri).get_storage_index()
        si_dir = storage_index_to_dir(si)
        sh0_file = [sharefile
                    for (shnum, serverid, sharefile)
                    in self.find_uri_shares(immutable_uri)
                    if shnum == 0][0]
        sh0_data = open(sh0_file, "rb").read()
        for clientnum in immutable_shares:
            if 0 in immutable_shares[clientnum]:
                continue
            cdir = self.get_serverdir(clientnum)
            target = os.path.join(cdir, "shares", si_dir, "0")
            outf = open(target, "wb")
            outf.write(sh0_data)
            outf.close()

        d = self.download_immutable()
        return d

    def test_verifycap(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]
        self.load_shares()

        n = self.c0.create_node_from_uri(immutable_uri)
        vcap = n.get_verify_cap().to_string()
        vn = self.c0.create_node_from_uri(vcap)
        d = download_to_data(vn)
        def _got_ciphertext(ciphertext):
            self.failUnlessEqual(len(ciphertext), len(plaintext))
            self.failIfEqual(ciphertext, plaintext)
        d.addCallback(_got_ciphertext)
        return d

class BrokenDecoder(CRSDecoder):
    def decode(self, shares, shareids):
        d = CRSDecoder.decode(self, shares, shareids)
        def _decoded(buffers):
            def _corruptor(s, which):
                return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:]
            buffers[0] = _corruptor(buffers[0], 0) # flip lsb of first byte
            return buffers
        d.addCallback(_decoded)
        return d


class PausingConsumer(MemoryConsumer):
    def __init__(self):
        MemoryConsumer.__init__(self)
        self.size = 0
        self.writes = 0
    def write(self, data):
        self.size += len(data)
        self.writes += 1
        if self.writes <= 2:
            # we happen to use 4 segments, and want to avoid pausing on the
            # last one (since then the _unpause timer will still be running)
            self.producer.pauseProducing()
            reactor.callLater(0.1, self._unpause)
        return MemoryConsumer.write(self, data)
    def _unpause(self):
        self.producer.resumeProducing()

class PausingAndStoppingConsumer(PausingConsumer):
    debug_stopped = False
    def write(self, data):
        if self.debug_stopped:
            raise Exception("I'm stopped, don't write to me")
        self.producer.pauseProducing()
        eventually(self._stop)
    def _stop(self):
        self.debug_stopped = True
        self.producer.stopProducing()

class StoppingConsumer(PausingConsumer):
    def write(self, data):
        self.producer.stopProducing()

class ImmediatelyStoppingConsumer(MemoryConsumer):
    def registerProducer(self, p, streaming):
        MemoryConsumer.registerProducer(self, p, streaming)
        self.producer.stopProducing()

class StallingConsumer(MemoryConsumer):
    def __init__(self, halfway_cb):
        MemoryConsumer.__init__(self)
        self.halfway_cb = halfway_cb
        self.writes = 0
    def write(self, data):
        self.writes += 1
        if self.writes == 1:
            self.halfway_cb()
        return MemoryConsumer.write(self, data)

class Corruption(_Base, unittest.TestCase):

    def _corrupt_flip(self, ign, imm_uri, which):
        log.msg("corrupt %d" % which)
        def _corruptor(s, debug=False):
            return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:]
        self.corrupt_shares_numbered(imm_uri, [2], _corruptor)

    def _corrupt_set(self, ign, imm_uri, which, newvalue):
        # type: (Any, bytes, int, int) -> None
        """
        Replace a single byte share file number 2 for the given capability with a
        new byte.

        :param imm_uri: Corrupt share number 2 belonging to this capability.
        :param which: The byte position to replace.
        :param newvalue: The new byte value to set in the share.
        """
        log.msg("corrupt %d" % which)
        def _corruptor(s, debug=False):
            return s[:which] + bchr(newvalue) + s[which+1:]
        self.corrupt_shares_numbered(imm_uri, [2], _corruptor)

    def test_each_byte(self):
        """
        Test share selection behavior of the downloader in the face of certain
        kinds of data corruption.

        1. upload a small share to the no-network grid
        2. read all of the resulting share files out of the no-network storage servers
        3. for each of

           a. each byte of the share file version field
           b. each byte of the immutable share version field
           c. each byte of the immutable share data offset field
           d. the most significant byte of the block_shares offset field
           e. one of the bytes of one of the merkle trees
           f. one of the bytes of the share hashes list

           i. flip the least significant bit in all of the the share files
           ii. perform the download/check/restore process

        4. add 2 ** 24 to the share file version number
        5. perform the download/check/restore process

        6. add 2 ** 24 to the share version number
        7. perform the download/check/restore process

        The download/check/restore process is:

        1. attempt to download the data
        2. assert that the recovered plaintext is correct
        3. assert that only the "correct" share numbers were used to reconstruct the plaintext
        4. restore all of the share files to their pristine condition
        """
        # Setting catalog_detection=True performs an exhaustive test of the
        # Downloader's response to corruption in the lsb of each byte of the
        # 2070-byte share, with two goals: make sure we tolerate all forms of
        # corruption (i.e. don't hang or return bad data), and make a list of
        # which bytes can be corrupted without influencing the download
        # (since we don't need every byte of the share). That takes 50s to
        # run on my laptop and doesn't have any actual asserts, so we don't
        # normally do that.
        self.catalog_detection = False

        self.basedir = "download/Corruption/each_byte"
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # to exercise the block-hash-tree code properly, we need to have
        # multiple segments. We don't tell the downloader about the different
        # segsize, so it guesses wrong and must do extra roundtrips.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 120 # 3 segs, 4-wide hashtree

        if self.catalog_detection:
            undetected = spans.Spans()

        def _download(ign, imm_uri, which, expected):
            n = self.c0.create_node_from_uri(imm_uri)
            n._cnode._maybe_create_download_node()
            # for this test to work, we need to have a new Node each time.
            # Make sure the NodeMaker's weakcache hasn't interfered.
            assert not n._cnode._node._shares
            d = download_to_data(n)
            def _got_data(data):
                self.failUnlessEqual(data, plaintext)
                shnums = sorted([s._shnum for s in n._cnode._node._shares])
                no_sh2 = bool(2 not in shnums)
                sh2 = [s for s in n._cnode._node._shares if s._shnum == 2]
                sh2_had_corruption = False
                if sh2 and sh2[0].had_corruption:
                    sh2_had_corruption = True
                num_needed = len(n._cnode._node._shares)
                if self.catalog_detection:
                    detected = no_sh2 or sh2_had_corruption or (num_needed!=3)
                    if not detected:
                        undetected.add(which, 1)
                if expected == "no-sh2":
                    self.failIfIn(2, shnums)
                elif expected == "2bad-need-3":
                    self.failIf(no_sh2)
                    self.failUnless(sh2[0].had_corruption)
                    self.failUnlessEqual(num_needed, 3)
                elif expected == "need-4th":
                    # XXX check with warner; what relevance does this
                    # have for the "need-4th" stuff?
                    #self.failIf(no_sh2)
                    #self.failUnless(sh2[0].had_corruption)
                    self.failIfEqual(num_needed, 3)
            d.addCallback(_got_data)
            return d


        d = self.c0.upload(u)
        def _uploaded(ur):
            imm_uri = ur.get_uri()
            self.shares = self.copy_shares(imm_uri)
            d = defer.succeed(None)
            # 'victims' is a list of corruption tests to run. Each one flips
            # the low-order bit of the specified offset in the share file (so
            # offset=0 is the MSB of the container version, offset=15 is the
            # LSB of the share version, offset=24 is the MSB of the
            # data-block-offset, and offset=48 is the first byte of the first
            # data-block). Each one also specifies what sort of corruption
            # we're expecting to see.
            no_sh2_victims = [0,1,2,3] # container version
            need3_victims =  [ ] # none currently in this category
            # when the offsets are corrupted, the Share will be unable to
            # retrieve the data it wants (because it thinks that data lives
            # off in the weeds somewhere), and Share treats DataUnavailable
            # as abandon-this-share, so in general we'll be forced to look
            # for a 4th share.
            need_4th_victims = [12,13,14,15, # offset[data]
                                24,25,26,27, # offset[block_hashes]
                                ]
            need_4th_victims.append(36) # block data
            # when corrupting hash trees, we must corrupt a value that isn't
            # directly set from somewhere else. Since we download data from
            # seg2, corrupt something on its hash chain, like [2] (the
            # right-hand child of the root)
            need_4th_victims.append(600+2*32) # block_hashes[2]
            # Share.loop is pretty conservative: it abandons the share at the
            # first sign of corruption. It doesn't strictly need to be this
            # way: if the UEB were corrupt, we could still get good block
            # data from that share, as long as there was a good copy of the
            # UEB elsewhere. If this behavior is relaxed, then corruption in
            # the following fields (which are present in multiple shares)
            # should fall into the "need3_victims" case instead of the
            # "need_4th_victims" case.
            need_4th_victims.append(824) # share_hashes
            corrupt_me = ([(i,"no-sh2") for i in no_sh2_victims] +
                          [(i, "2bad-need-3") for i in need3_victims] +
                          [(i, "need-4th") for i in need_4th_victims])
            if self.catalog_detection:
                share_len = len(list(self.shares.values())[0])
                corrupt_me = [(i, "") for i in range(share_len)]
                # This is a work around for ticket #2024.
                corrupt_me = corrupt_me[0:8]+corrupt_me[12:]
            for i,expected in corrupt_me:
                # All these tests result in a successful download. What we're
                # measuring is how many shares the downloader had to use.
                d.addCallback(self._corrupt_flip, imm_uri, i)
                d.addCallback(_download, imm_uri, i, expected)
                d.addCallback(lambda ign: self.restore_all_shares(self.shares))
                d.addCallback(fireEventually)
            corrupt_values = [
                # Make the container version for share number 2 look
                # unsupported.  If you add support for immutable share file
                # version number much past 16 million then you will have to
                # update this test.  Also maybe you have other problems.
                (1, 255, "no-sh2"),
                # Make the immutable share number 2 (not the container, the
                # thing inside the container) look unsupported.  Ditto the
                # above about version numbers in the ballpark of 16 million.
                (13, 255, "need-4th"),
            ]
            for i,newvalue,expected in corrupt_values:
                d.addCallback(self._corrupt_set, imm_uri, i, newvalue)
                d.addCallback(_download, imm_uri, i, expected)
                d.addCallback(lambda ign: self.restore_all_shares(self.shares))
                d.addCallback(fireEventually)
            return d
        d.addCallback(_uploaded)
        def _show_results(ign):
            share_len = len(list(self.shares.values())[0])
            print()
            print("of [0:%d], corruption ignored in %s" %
                   (share_len, undetected.dump()))
        if self.catalog_detection:
            d.addCallback(_show_results)
            # of [0:2070], corruption ignored in len=1133:
            # [4-11],[16-23],[28-31],[152-439],[600-663],[1309-2069]
            #  [4-11]: container sizes
            #  [16-23]: share block/data sizes
            #  [152-375]: plaintext hash tree
            #  [376-408]: crypttext_hash_tree[0] (root)
            #  [408-439]: crypttext_hash_tree[1] (computed)
            #  [600-631]: block hash tree[0] (root)
            #  [632-663]: block hash tree[1] (computed)
            #  [1309-]: reserved+unused UEB space
        return d

    def test_failure(self):
        # this test corrupts all shares in the same way, and asserts that the
        # download fails.

        self.basedir = "download/Corruption/failure"
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # to exercise the block-hash-tree code properly, we need to have
        # multiple segments. We don't tell the downloader about the different
        # segsize, so it guesses wrong and must do extra roundtrips.
        u = upload.Data(plaintext, None)
        u.max_segment_size = 120 # 3 segs, 4-wide hashtree

        d = self.c0.upload(u)
        def _uploaded(ur):
            imm_uri = ur.get_uri()
            self.shares = self.copy_shares(imm_uri)

            corrupt_me = [(48, "block data", "Last failure: None"),
                          (600+2*32, "block_hashes[2]", "BadHashError"),
                          (376+2*32, "crypttext_hash_tree[2]", "BadHashError"),
                          (824, "share_hashes", "BadHashError"),
                          ]
            def _download(imm_uri):
                n = self.c0.create_node_from_uri(imm_uri)
                n._cnode._maybe_create_download_node()
                # for this test to work, we need to have a new Node each time.
                # Make sure the NodeMaker's weakcache hasn't interfered.
                assert not n._cnode._node._shares
                return download_to_data(n)

            d = defer.succeed(None)
            for i,which,substring in corrupt_me:
                # All these tests result in a failed download.
                d.addCallback(self._corrupt_flip_all, imm_uri, i)
                d.addCallback(lambda ign, which=which, substring=substring:
                              self.shouldFail(NoSharesError, which,
                                              substring,
                                              _download, imm_uri))
                d.addCallback(lambda ign: self.restore_all_shares(self.shares))
                d.addCallback(fireEventually)
            return d
        d.addCallback(_uploaded)

        return d

    def _corrupt_flip_all(self, ign: Any, imm_uri: bytes, which: int) -> None:
        """
        Flip the least significant bit at a given byte position in all share files
        for the given capability.
        """
        def _corruptor(s, debug=False):
            # type: (bytes, bool) -> bytes
            before_corruption = s[:which]
            after_corruption = s[which+1:]
            original_byte = s[which:which+1]
            corrupt_byte = bchr(ord(original_byte) ^ 0x01)
            return b"".join([before_corruption, corrupt_byte, after_corruption])
        self.corrupt_all_shares(imm_uri, _corruptor)

class DownloadV2(_Base, unittest.TestCase):
    # tests which exercise v2-share code. They first upload a file with
    # FORCE_V2 set.

    def setUp(self):
        d = defer.maybeDeferred(_Base.setUp, self)
        def _set_force_v2(ign):
            self.old_force_v2 = layout.FORCE_V2
            layout.FORCE_V2 = True
        d.addCallback(_set_force_v2)
        return d
    def tearDown(self):
        layout.FORCE_V2 = self.old_force_v2
        return _Base.tearDown(self)

    def test_download(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # upload a file
        u = upload.Data(plaintext, None)
        d = self.c0.upload(u)
        def _uploaded(ur):
            imm_uri = ur.get_uri()
            n = self.c0.create_node_from_uri(imm_uri)
            return download_to_data(n)
        d.addCallback(_uploaded)
        return d

    def test_download_no_overrun(self):
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        # tweak the client's copies of server-version data, so it believes
        # that they're old and can't handle reads that overrun the length of
        # the share. This exercises a different code path.
        for s in self.c0.storage_broker.get_connected_servers():
            v = s.get_version()
            v1 = v[b"http://allmydata.org/tahoe/protocols/storage/v1"]
            v1[b"tolerates-immutable-read-overrun"] = False

        # upload a file
        u = upload.Data(plaintext, None)
        d = self.c0.upload(u)
        def _uploaded(ur):
            imm_uri = ur.get_uri()
            n = self.c0.create_node_from_uri(imm_uri)
            return download_to_data(n)
        d.addCallback(_uploaded)
        return d

    def OFF_test_no_overrun_corrupt_shver(self): # unnecessary
        self.basedir = self.mktemp()
        self.set_up_grid()
        self.c0 = self.g.clients[0]

        for s in self.c0.storage_broker.get_connected_servers():
            v = s.get_version()
            v1 = v["http://allmydata.org/tahoe/protocols/storage/v1"]
            v1["tolerates-immutable-read-overrun"] = False

        # upload a file
        u = upload.Data(plaintext, None)
        d = self.c0.upload(u)
        def _uploaded(ur):
            imm_uri = ur.get_uri()
            def _do_corrupt(which, newvalue):
                def _corruptor(s, debug=False):
                    return s[:which] + chr(newvalue) + s[which+1:]
                self.corrupt_shares_numbered(imm_uri, [0], _corruptor)
            _do_corrupt(12+3, 0x00)
            n = self.c0.create_node_from_uri(imm_uri)
            d = download_to_data(n)
            def _got_data(data):
                self.failUnlessEqual(data, plaintext)
            d.addCallback(_got_data)
            return d
        d.addCallback(_uploaded)
        return d

class Status(unittest.TestCase):
    def test_status(self):
        now = 12345.1
        ds = DownloadStatus("si-1", 123)
        self.failUnlessEqual(ds.get_status(), "idle")
        ev0 = ds.add_segment_request(0, now)
        self.failUnlessEqual(ds.get_status(), "fetching segment 0")
        ev0.activate(now+0.5)
        ev0.deliver(now+1, 0, 1000, 2.0)
        self.failUnlessEqual(ds.get_status(), "idle")
        ev2 = ds.add_segment_request(2, now+2)
        del ev2 # hush pyflakes
        ev1 = ds.add_segment_request(1, now+2)
        self.failUnlessEqual(ds.get_status(), "fetching segments 1,2")
        ev1.error(now+3)
        self.failUnlessEqual(ds.get_status(),
                             "fetching segment 2; errors on segment 1")

    def test_progress(self):
        now = 12345.1
        ds = DownloadStatus("si-1", 123)
        self.failUnlessEqual(ds.get_progress(), 0.0)
        e = ds.add_read_event(0, 1000, now)
        self.failUnlessEqual(ds.get_progress(), 0.0)
        e.update(500, 2.0, 2.0)
        self.failUnlessEqual(ds.get_progress(), 0.5)
        e.finished(now+2)
        self.failUnlessEqual(ds.get_progress(), 1.0)

        e1 = ds.add_read_event(1000, 2000, now+3)
        e2 = ds.add_read_event(4000, 2000, now+3)
        self.failUnlessEqual(ds.get_progress(), 0.0)
        e1.update(1000, 2.0, 2.0)
        self.failUnlessEqual(ds.get_progress(), 0.25)
        e2.update(1000, 2.0, 2.0)
        self.failUnlessEqual(ds.get_progress(), 0.5)
        e1.update(1000, 2.0, 2.0)
        e1.finished(now+4)
        # now there is only one outstanding read, and it is 50% done
        self.failUnlessEqual(ds.get_progress(), 0.5)
        e2.update(1000, 2.0, 2.0)
        e2.finished(now+5)
        self.failUnlessEqual(ds.get_progress(), 1.0)

    def test_active(self):
        now = 12345.1
        ds = DownloadStatus("si-1", 123)
        self.failUnlessEqual(ds.get_active(), False)
        e1 = ds.add_read_event(0, 1000, now)
        self.failUnlessEqual(ds.get_active(), True)
        e2 = ds.add_read_event(1, 1000, now+1)
        self.failUnlessEqual(ds.get_active(), True)
        e1.finished(now+2)
        self.failUnlessEqual(ds.get_active(), True)
        e2.finished(now+3)
        self.failUnlessEqual(ds.get_active(), False)

def make_server(clientid):
    tubid = hashutil.tagged_hash(b"clientid", clientid)[:20]
    return NoNetworkServer(tubid, None)
def make_servers(clientids):
    servers = {}
    for clientid in clientids:
        servers[clientid] = make_server(clientid)
    return servers

class MyShare(object):
    def __init__(self, shnum, server, rtt):
        self._shnum = shnum
        self._server = server
        self._dyhb_rtt = rtt

    def __repr__(self):
        return "sh%d-on-%s" % (self._shnum, str(self._server.get_name(), "ascii"))

class MySegmentFetcher(SegmentFetcher):
    def __init__(self, *args, **kwargs):
        SegmentFetcher.__init__(self, *args, **kwargs)
        self._test_start_shares = []
    def _start_share(self, share, shnum):
        self._test_start_shares.append(share)

class FakeNode(object):
    def __init__(self):
        self.want_more = 0
        self.failed = None
        self.processed = None
        self._si_prefix = "si_prefix"

    def want_more_shares(self):
        self.want_more += 1

    def fetch_failed(self, fetcher, f):
        self.failed = f

    def process_blocks(self, segnum, blocks):
        self.processed = (segnum, blocks)

    def get_num_segments(self):
        return 1, True


class Selection(unittest.TestCase):
    def test_failure(self):
        """If the fetch loop fails, it tell the Node the fetch failed."""
        node = FakeNode()
        # Simulate a failure:
        node.get_num_segments = lambda: 1/0
        sf = SegmentFetcher(node, 0, 3, None)
        sf.add_shares([])
        d = flushEventualQueue()
        def _check1(ign):
            [_] = self.flushLoggedErrors(ZeroDivisionError)
            self.failUnless(node.failed)
            self.failUnless(node.failed.check(ZeroDivisionError))
        d.addCallback(_check1)
        return d

    def test_no_shares(self):
        node = FakeNode()
        sf = SegmentFetcher(node, 0, 3, None)
        sf.add_shares([])
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            self.failUnlessEqual(node.failed, None)
            sf.no_more_shares()
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnless(node.failed)
            self.failUnless(node.failed.check(NoSharesError))
        d.addCallback(_check2)
        return d

    def test_only_one_share(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        serverA = make_server(b"peer-A")
        shares = [MyShare(0, serverA, 0.0)]
        sf.add_shares(shares)
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            self.failUnlessEqual(node.failed, None)
            sf.no_more_shares()
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnless(node.failed)
            self.failUnless(node.failed.check(NotEnoughSharesError))
            sname = serverA.get_name()
            self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused="  % str(sname, "ascii"),
                              str(node.failed))
        d.addCallback(_check2)
        return d

    def test_good_diversity_early(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        shares = [MyShare(i, make_server(b"peer-%d" % i), i) for i in range(10)]
        sf.add_shares(shares)
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 0)
            self.failUnlessEqual(sf._test_start_shares, shares[:3])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      1: "block-1",
                                                      2: "block-2"}) )
        d.addCallback(_check2)
        return d

    def test_good_diversity_late(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        shares = [MyShare(i, make_server(b"peer-%d" % i), i) for i in range(10)]
        sf.add_shares([])
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            sf.add_shares(shares)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnlessEqual(sf._test_start_shares, shares[:3])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      1: "block-1",
                                                      2: "block-2"}) )
        d.addCallback(_check3)
        return d

    def test_avoid_bad_diversity_late(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        # we could satisfy the read entirely from the first server, but we'd
        # prefer not to. Instead, we expect to only pull one share from the
        # first server
        servers = make_servers([b"peer-A", b"peer-B", b"peer-C"])
        shares = [MyShare(0, servers[b"peer-A"], 0.0),
                  MyShare(1, servers[b"peer-A"], 0.0),
                  MyShare(2, servers[b"peer-A"], 0.0),
                  MyShare(3, servers[b"peer-B"], 1.0),
                  MyShare(4, servers[b"peer-C"], 2.0),
                  ]
        sf.add_shares([])
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            sf.add_shares(shares)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnlessEqual(sf._test_start_shares,
                                 [shares[0], shares[3], shares[4]])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      3: "block-3",
                                                      4: "block-4"}) )
        d.addCallback(_check3)
        return d

    def test_suffer_bad_diversity_late(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        # we satisfy the read entirely from the first server because we don't
        # have any other choice.
        serverA = make_server(b"peer-A")
        shares = [MyShare(0, serverA, 0.0),
                  MyShare(1, serverA, 0.0),
                  MyShare(2, serverA, 0.0),
                  MyShare(3, serverA, 0.0),
                  MyShare(4, serverA, 0.0),
                  ]
        sf.add_shares([])
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            sf.add_shares(shares)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnlessEqual(node.want_more, 3)
            self.failUnlessEqual(sf._test_start_shares,
                                 [shares[0], shares[1], shares[2]])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      1: "block-1",
                                                      2: "block-2"}) )
        d.addCallback(_check3)
        return d

    def test_suffer_bad_diversity_early(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        # we satisfy the read entirely from the first server because we don't
        # have any other choice.
        serverA = make_server(b"peer-A")
        shares = [MyShare(0, serverA, 0.0),
                  MyShare(1, serverA, 0.0),
                  MyShare(2, serverA, 0.0),
                  MyShare(3, serverA, 0.0),
                  MyShare(4, serverA, 0.0),
                  ]
        sf.add_shares(shares)
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 2)
            self.failUnlessEqual(sf._test_start_shares,
                                 [shares[0], shares[1], shares[2]])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      1: "block-1",
                                                      2: "block-2"}) )
        d.addCallback(_check2)
        return d

    def test_overdue(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        shares = [MyShare(i, make_server(b"peer-%d" % i), i) for i in range(10)]
        sf.add_shares(shares)
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 0)
            self.failUnlessEqual(sf._test_start_shares, shares[:3])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, OVERDUE)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnlessEqual(sf._test_start_shares, shares[:6])
            for sh in sf._test_start_shares[3:]:
                sf._block_request_activity(sh, sh._shnum, COMPLETE,
                                           "block-%d" % sh._shnum)
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {3: "block-3",
                                                      4: "block-4",
                                                      5: "block-5"}) )
        d.addCallback(_check3)
        return d

    def test_overdue_fails(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        servers = make_servers([b"peer-%d" % i for i in range(6)])
        shares = [MyShare(i, servers[b"peer-%d" % i], i) for i in range(6)]
        sf.add_shares(shares)
        sf.no_more_shares()
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 0)
            self.failUnlessEqual(sf._test_start_shares, shares[:3])
            for sh in sf._test_start_shares:
                sf._block_request_activity(sh, sh._shnum, OVERDUE)
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            self.failUnlessEqual(sf._test_start_shares, shares[:6])
            for sh in sf._test_start_shares[3:]:
                sf._block_request_activity(sh, sh._shnum, DEAD)
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            # we're still waiting
            self.failUnlessEqual(node.processed, None)
            self.failUnlessEqual(node.failed, None)
            # now complete one of the overdue ones, and kill one of the other
            # ones, leaving one hanging. This should trigger a failure, since
            # we cannot succeed.
            live = sf._test_start_shares[0]
            die = sf._test_start_shares[1]
            sf._block_request_activity(live, live._shnum, COMPLETE, "block")
            sf._block_request_activity(die, die._shnum, DEAD)
            return flushEventualQueue()
        d.addCallback(_check3)
        def _check4(ign):
            self.failUnless(node.failed)
            self.failUnless(node.failed.check(NotEnoughSharesError))
            sname = servers[b"peer-2"].get_name()
            self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % str(sname, "ascii"),
                              str(node.failed))
        d.addCallback(_check4)
        return d

    def test_avoid_redundancy(self):
        node = FakeNode()
        sf = MySegmentFetcher(node, 0, 3, None)
        # we could satisfy the read entirely from the first server, but we'd
        # prefer not to. Instead, we expect to only pull one share from the
        # first server
        servers = make_servers([b"peer-A", b"peer-B", b"peer-C", b"peer-D",
                                b"peer-E"])
        shares = [MyShare(0, servers[b"peer-A"],0.0),
                  MyShare(1, servers[b"peer-B"],1.0),
                  MyShare(0, servers[b"peer-C"],2.0), # this will be skipped
                  MyShare(1, servers[b"peer-D"],3.0),
                  MyShare(2, servers[b"peer-E"],4.0),
                  ]
        sf.add_shares(shares[:3])
        d = flushEventualQueue()
        def _check1(ign):
            self.failUnlessEqual(node.want_more, 1)
            self.failUnlessEqual(sf._test_start_shares,
                                 [shares[0], shares[1]])
            # allow sh1 to retire
            sf._block_request_activity(shares[1], 1, COMPLETE, "block-1")
            return flushEventualQueue()
        d.addCallback(_check1)
        def _check2(ign):
            # and then feed in the remaining shares
            sf.add_shares(shares[3:])
            sf.no_more_shares()
            return flushEventualQueue()
        d.addCallback(_check2)
        def _check3(ign):
            self.failUnlessEqual(sf._test_start_shares,
                                 [shares[0], shares[1], shares[4]])
            sf._block_request_activity(shares[0], 0, COMPLETE, "block-0")
            sf._block_request_activity(shares[4], 2, COMPLETE, "block-2")
            return flushEventualQueue()
        d.addCallback(_check3)
        def _check4(ign):
            self.failIfEqual(node.processed, None)
            self.failUnlessEqual(node.processed, (0, {0: "block-0",
                                                      1: "block-1",
                                                      2: "block-2"}) )
        d.addCallback(_check4)
        return d
