From f6df94a50efdc582b26c98a1ccecf1a7466da22a Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Mon, 30 Nov 2020 07:25:10 +0100 Subject: [PATCH 1/2] prune: Add self-healing Allow prune to heal situations where blobs in the index are missing or the corresponding packfiles are damaged if those blobs are not needed. --- cmd/restic/cmd_prune.go | 36 ++++++++++++++++-- cmd/restic/integration_test.go | 8 +++- .../testdata/repo-unused-data-missing.tar.gz | Bin 0 -> 8979 bytes 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 cmd/restic/testdata/repo-unused-data-missing.tar.gz diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index c8c665cde..8ee635f64 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -310,7 +310,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB return nil } - if p.unusedSize+p.usedSize != uint64(packSize) { + if p.unusedSize+p.usedSize != uint64(packSize) && + !(p.usedBlobs == 0 && p.duplicateBlobs == 0) { + // Pack size does not fit and pack is needed => error + // If the pack is not needed, this is no error, the pack can + // and will be simply removed, see below. Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.", id.Str(), p.unusedSize+p.usedSize, packSize) return errorSizeNotMatching @@ -356,10 +360,26 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB return err } + // At this point indexPacks contains only missing packs! + + // missing packs that are not needed can be ignored + ignorePacks := restic.NewIDSet() + for id, p := range indexPack { + if p.usedBlobs == 0 && p.duplicateBlobs == 0 { + ignorePacks.Insert(id) + stats.blobs.remove += p.unusedBlobs + stats.size.remove += p.unusedSize + delete(indexPack, id) + } + } + if len(indexPack) != 0 { - Warnf("The index references pack files which are missing from the repository: %v\n", indexPack) + Warnf("The index references needed pack files which are missing from the repository: %v\n", indexPack) return errorPacksMissing } + if len(ignorePacks) != 0 { + Verbosef("missing but unneded pack files are referenced in the index, will be repaired\n") + } repackAllPacksWithDuplicates := true @@ -492,12 +512,20 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB removePacks.Merge(repackPacks) } - if len(removePacks) != 0 { - err = rebuildIndexFiles(gopts, repo, removePacks, nil) + if len(ignorePacks) == 0 { + ignorePacks = removePacks + } else { + ignorePacks.Merge(removePacks) + } + + if len(ignorePacks) != 0 { + err = rebuildIndexFiles(gopts, repo, ignorePacks, nil) if err != nil { return err } + } + if len(removePacks) != 0 { Verbosef("removing %d old packs\n", len(removePacks)) DeleteFiles(gopts, repo, removePacks, restic.PackFile) } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index be8ec2bd9..7cfd05f6a 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -1502,10 +1502,16 @@ func TestEdgeCaseRepos(t *testing.T) { // repo where a blob is missing // => check and prune should fail - t.Run("no-data", func(t *testing.T) { + t.Run("missing-data", func(t *testing.T) { testEdgeCaseRepo(t, "repo-data-missing.tar.gz", opts, pruneDefaultOptions, false, false) }) + // repo where blobs which are not needed are missing or in invalid pack files + // => check should fail and prune should repair this + t.Run("missing-unused-data", func(t *testing.T) { + testEdgeCaseRepo(t, "repo-unused-data-missing.tar.gz", opts, pruneDefaultOptions, false, true) + }) + // repo where data exists that is not referenced // => check and prune should fully work t.Run("unreferenced-data", func(t *testing.T) { diff --git a/cmd/restic/testdata/repo-unused-data-missing.tar.gz b/cmd/restic/testdata/repo-unused-data-missing.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..91b9702b872671430d5b0daa1754d14ddd303450 GIT binary patch literal 8979 zcmZ9Qc|26#AOFWvT72xKBBPQuYeb@(YzfJpR3nKHvS-N^m82S3lC80eLe}gfYa(M` zvPMkyWvsK@^Si#k-(SDSWBxdE@0s^G@AdI~4K`vQ=0-##4#TpL#Ae1f`@Tg(-bUxT zJeM68Hf4uL&`TsqmB;xhEq zt|MI09rwRJc!<%Ei)o$y`VXhp@~LhgCQuLNs((kE#lM~7)Q}4amtfYq6-+EgdR9Km z92@dGci+CLh!i0kt9!M~C4R2wux0V;AtTQpr_53(l(x4=%aayj{Ja(T!}5-1`Lys_ zLx%e`hSNfMGrStBr{y|rWjEW>v>kQX`E;GeRgJa^9w+HFLMccZf}DX)s^k77&Wp0zy@m7Ugl+ja(=_`DcZrtvXN!KZ@`0q!e$(?}CpA~tQ=6z4t_jXtd27Vnf zv|;HTnKV6lNdpC6?b}1n3gtwNY;yYv2VKwnW^@)a^HY4TH-sg9| z>0px8HPZ;QtK^QO^=2cd2G z`zKqtePFHW5X~X)RQKVr(Sle0Ygtu~!4H{HZ=Kc6vL_ZS5vr1wSvnWgJS~Y%fmKE3 z`ufOv>T4;T(49&3u3^T#1pFcGLcwL?+^yuKl;hElxArghVeFHm%f0YtsV6VWB;yI| z%X9Kkv2k7k-v);AKHbMEPi@~MDTbFHTV#$u{ifjQ*<$?&{%NJtYcCELQ;RILNJ;)d zPb^!X30$jIrNl}G%h26RI}S?c-!5!Et6NIRt7|>$8%Fk3QvFZyh^}MqukEoi z3Si;}q~PawK87~PMd-5~QRK5vJM+4EqT5kUhvf#><$GWMoq7HCgn=mU%&EYe5hl#aBo2&N6HE99Ypot{^URqM{(ML-xuda`Jlg+;N4BI{ zy~J6?&MXhHrLKH*%(W`bj$QUCHB4RY=c8R5@6WnE!hJ+eJxJbUPCS6_PN!YYpXpv{ zDSfUv88+QA70(&q+jpaBTCICLLBaZHmtQHVK6t@ol8-z<+#Ph&P3o##4b5ASi$TBx3y{+A#e6gVE-D$CpOO^|*ssdeI#^XT*DcYk|U6%#if z#QIp6Xa>GYX}of3nbWHLkH1kyr;&6llciK>`g-d5psRM{XxwFw*>aic>%HBNQLak8fQqUAp-4 zc>Ar*jQJfK-6zX7Pd1Ac%HnLidkuS>HGMqQ>rF1#m)@DMx7c%$Bhg;%-^y<_uUGv8 ziNf-pzlycv^AE9wtnaM}8Fg4OOal8D8D85qcJ7Ya`F}g4g+wJpwVAXFC~^8O7w?uS z$2+)-ShMuF8E4>^AO3#ndi^|?h4GB7SzM2gve&hq^qH*HLC6lAk4$xjmAx zGa|ztH|`A#1uL6)yd_Ex)^jPVS(ut}Gq%y*)-*1*__wacj};sV^z>`YZC%eY5KG@0 z>X`c{a_V-(pH7X9i2JdIq@*@CIsG#uQBrOj8E&es-Y4!otqGtHYx(Np|I-Y&3>yq} zir1S{DxQ_!&oAml8hEY~8OK-H*#Rwi}xR zSCo3L`4dC~uFaF&cRmicu%Y<|^n_Tu}V*UOv$736pJ#X;p6B zWJB%Y&dMDTqwij{;m_Z*re1t6;~!?vV-CfZjQJgD*!p#&l2x_jgc?u6exT(Rb$h+! z$C!WO4Zo%em+o3)9U6nR6U}49>BH4q`g$Y>r{} zA3w|Ma6*{v`G;Gt-xS_ejo2gaD12Pks$KTuloW#7W(J8sSNL%+Navac`%{a(Yig@! z(r4UrsG*&uBiTcjs%4{wAAu@)vqrOX=gnw zaG1-gyfB!$LlHZ2t%+3O<>RfBT$946fe@S!H;w;=r{dwTJ?`AB(+BbdgEj7+<6u4# z(U#%Cl?*IQY66!Vs1ae0BZ<5Op)UqtNGxEVBzP|B`-Y`N7h-<7^< z`rc@B&WiJJbM=7@!-TW!3qVjKc zez1`@6SsgxUXo{{SCOb^W)h{i|8M_oh)^mvIBVdkk&E6Afh5YJcZ> zS8)G&pTHe!$5mt6l51$Yebde?$G&RxVg>Cv#bR_|OSb00gPX+*zkjrgB5ncW_RVIs zRYCT@Ln6-x3Kh=829{;s4Cl3uoR-lbG zC^qOW>nj+ZuDed?)QeUfG7&T0u&$9zA2iEZY)#HAHjmXj%@J`vL+JUr!)?Vs2Q1%s zn)+?!26w)TE%ye=OHA)15l}LHkmfFp8D!JxApP?0a-HSvXjx zC+nL8o*otCSfBQMec<(pAN(6F;0we$SuYoS6YQT5J8gjp`%eBETN3+h=lEB_x;PfE z0IuE_cU`#ppD|q*R}D#>!({x2xq9qs$faw+vJGLU=%HQCC)FmG_{yg=wu5QUg1I8JyXgMRD$oJV&xo)l%huzO^o^$7Z za6TdRqoStrfba!TY{t%f{w)bLsl@LN`$#t5C;X|Z8cX&s-U}zY894+{A0OLxPreme zGc?A|$;-m2evEZz<%8?v51%o<>+Fw|Ru!97K3HM|R=@QrWU@^LzI|FJ_i}lP>&4vs zkl%05-T!2BM8(7iW$Z9zzQK(An>hceA;=F^ZN>c z0lhHUG}>1b@&fzzOgz7G22`$*`Zk`|uCmYguE&B78>Mb&Sfx)@SJByW>F4>Xs!Kts zv^r_dKFQjlU+pfMU7JIXzTDFGf3H<^)IF@-zskwDZ-y+#)&?l?gSARxMO+Tp6az4G`9jWS@pbf6f z^C!$@^={#lc2yNM0GB{yB`l5)`bs!B@cPnCyqryKk+^xL!HeY_E%5PU#;guwz#u}n>&7oWPLvva>JL^+! zjsry)8ICl$j%kJS4@H}qkHTAmxX+;B$YwSb``T8n6ybe3_Xj7|J~#bK{l=1KQ6+K9 z_t#KirLv;yywZYfp^G)p9_Fv@AvsXm#_ zw2?_GvY?}-)9rEWtFt3_Fz>B$s|EvKJGl8BTKNjX?YwCMABQ>h+q?7KH)C@4-g3N< z*i5l{A65R4xSyQfegLXS?&+`8Hg{~sB`kRM41YlaaNcVxLx^s%NKr2=yza>eO8}=| z$TJ*&e5>4p>|w}%KpP*OAush$GTb9`0Z;u0Uil_iG3i!P7EueaDp0x?!@M!ZNj zVGXd#XF+iWt5}ZuT?x(u{)lvALv!eOuI6=e3c8~R4mr|8xR$?xOl?VsT!rxA`1Nc$B>#FA zA6P7H1ELtF4EQp7)Q?gS_6og;2*UOw12P9E_HqNGEGl;I|2-cc<%Do%%@B8kC|WTC zi8Jq^!S7GK$mG7Hc$sU_+_Gg3^NQk zxxWHJFC8C6f)1o0>3_jJAj>m>WXA$`Mi}!4AY2Npn;AI1U=Y^3LtfrB4c%lw7lC*% zBAba2O@6NGGW~@!=){vW*cgMjnQUN5U=lL}=`|6^&?C$aB+VdDV7^Oqy*}X453XZ& zu$dGRjTnhp%~k>pThIYvknV+80d3}O7~Kjw$m|_ zMs1*l%6Xs;=sF`2Q+PBJX}y3<$L{|M!Yt@JjNLp+ECL(A#)EqO9wDK(pI~H{fbb36 zuHhVnch5(l!_*E(&h4g>1i~ zf8jGhWS=sK;us*7vrR!P1(DapXJ#Od2wtO>-mSrF2v+bCVlf|p4k6+q1LB4T9m?EsGwwieXeAjkkO-3;EgAl4xRJ00rLLIJ z`y?wmQY<2LjPnkgscK{zA2^V%Bg%g$m1OWB2_L|?H}4NZ}M z6|$=R7f~Xz-H>@lui9?=%QMur%8v{J;qTilbV3X8@!NihmF|!{PBl;V`4+I=Q|OS) z`}xbM%?HP6Pt@yjAK4ZKY>aOn7C1*s{(QbnW!}ED%>r*u;5drJ=~|BMES?RMqMXiW z*?a3r@lMu2)9+{HKX}vm-ots;iHV}@2RCRp`y0L$j<|N^R14v1J)En5*Z9xvOU$*{ zSiCK5q;T%qX7TxRV`Vv;Rl%kdd!Ji}*Ty4tLFl5Y&AL%R$dvl zwHUCBWym@BJ9ph^Iy;?rn;6IsRiuEyR%_-j4TmLL_6l9&V5hW>tDV^X`{AFbw?mme zOH{-pPtT0@=xAJ@Q)!Xscvsx!w>SfzqN_<8{x7-$zv~oV5Hie6OC$dj`{NnFI_~zd zKl8rrfPOg30i{yXh}%r>amtmXMfH{WPiGJ5`5@^*Uvk}iB!ZMA`@V0rYRLy#m_KZu zdlCH(YPrGSaVV``?EJpr?VJQjRQfkQ@_$Q!f4&*L@M^EMnSK=c%n>iy$`r5CfffDf z%obij>d1hz`TgUrzV^a%IqVmooCk)z3!2MJL5NBadSio4P?eBjA4lk=@__PO})ca>uyI$WHJVx3U(jdhDUeD62i499L@$(`LnGbqPNmzRBFa z0!n_4d`X{rJCj!xwdtU_DO=4wZ-3|G#H^wUB`Bh4sYsbKUV7;K7j?gc+^>FG-m`CG zQaUR=6SjMtPd>@yciX5O3uv$*bFOW-rFv7{GvrJi&Fy5JIKb@ef`DyE49x! zH}ywF*pKIyhQg1#ax7-AyN!%1`x?%7c$(kTud*(zFpl&7*}K+gRc5`{I_6(B5m&0- z@$0*q->j5?^$Y*XT)xsassAR+zAtK&xk&Hi3kdo8eCdG8U91O@v(f5M+}Ptl@Aa_4 zCOx48ttDj<$?tU*E6vULV=nd;38y=NeZ(U3#CxIiI}AHljSO;Gg7@a1p@_C>Sl(K3 zO((MU#D#@)1$?-R`Brgq72|P6yF<-%e8C%)`zzxb^RX~rH{5%A2%24m%bg~A6=^O;CsgXV%e;kIO=p=capKwGFO~g zG47&YkmT$qLzkA|k1jEHKBn*(?L#A}TDlG&@Ls-7%=U2Ldi08JcDx6q^>X_Fl;16Yv;w zhrgzqe^@cTKEC{H#=O4NPM3^G`Dgbwh8g%dn@{kcKFE8D>^D$b_Paa3-f?haDED9c zm2vS9;gF?i~5{AH_6h8vk2!+@MI{@TO1Y!22j(!ke10)_QL%zXExR+1%Wq+&u%= zJCn`Y_Ot3Erqb^VHe9Sv`f*v8L8n{fFB-&Fq!qyJl@|r5xc=?tw=SMSQA&OrbMLSm zl2jk#M!vYIN*3GCBGu)t7WJ=;EaqtCOZ}%=tL~hAJ*wnN_Za`?0w;9aEvRd{Snm{4 zn0BP7e```tbY0WM)8`z|4W!2Dwh3ofHe9QnT$MPeuQSu%c7T{@Z42Et?`_kbME*GI z_2}5|EXNazE;1iI z=n6?ug*FF9%R3oA>b?nFqdu+@wYrh3?rQ!kd2Oux>!k)&TG?6LIo$0d#|ps|IeAB& zKtr)Yua`YfqblOfoueHps?Tq?mQ>&Qc&*?;#Jcs#{w;xZzj^1Uajtg;g!XT5?p$B2 zA`xFLBo~$@HjPcN<+Jg2EFWET>G`GMp>>Yp)tf;1oM^E$pU9u=c-QqYd6<_!L|R~O zhJW}+%dKMh-!iRljpl!^&^HK0zo}Tc4Vtr49pZbHLL31a%D;3 z0{rN^coU=rfkjZp%K;5EC=$cK-W&%LH0+*85PuWwjDmv5@OmtnFomzzUsZl}EZm}i zpU>pE&fVUebq|dbJwHm1J=aQCx?-1hD-U)*JHH-c9P@KetU-=xxN4?&SMKz&3ImE; zP|GqC!b!mbL0BV_xGhhDg;99QSc9H!VEIs>(*wNs15rj>gG`Cw3y8;bf)nPuJjtQm zPe5W3o?eDdF?B#WbixK!uns|0g9C$~?K*h(6KL&1J5?5iUk2_<@D@P}Sc-D{AqTmy zgPsI1LBXSFeZ3prp^aY&UWR+R;R-~TsgV!`!ce$m9Rw%D4#D}mCOA z97OD@3v(dhhmvXvIEu0;>S!I}U_vHrz=CyjWeEm>MpOw#u=^PV2DUsH;gn|JDG(o; zRDJLVJ+>TVKAS9vc|QDx)1BSWCJ%Ht~0oXhW31#pU1XUwjTZ{sa z3UM-n__CT0uKA>NHG?OwL)?6RBjlFi53>*h``&$NpG@Y={NuaCV+G99EF_PK_*YoS z^hvxhReAjlvag9k_MyzfUg`MNz>~A>{T(n!6+(=rM z?bZV~WlG4;S09*?Jv){@=jjco)td1ZP6Z58zq-T?m%UclfMa*~gQ@sk)dHbKr7s1@ zsMzR$BpA4Q)YA!ibOMO{9S%e)n>k5@-v{iqs)SH&MNS+Na~Y5wS7w6fG&; zS%&3RKZJ7+T7q-n5rSv#1QJic(+vpSFKe=c-5m^M0@~HfuwXEV?xm3!I6+kAnINPM zEDt&#ya8ove0NVj!k@n@?g%*>0Gc80Iog8do&chm=(yQ!1Z9(DXc=y1>IE@laOQ4- zOwb_ztE&#E>;J2$! zoolcsni{Gs+6FXuHy$R877F$X24K-ATX5R0>PN8susendNdq Date: Mon, 30 Nov 2020 13:53:12 +0100 Subject: [PATCH 2/2] prune: Correct error message --- cmd/restic/cmd_prune.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 8ee635f64..bdad4efd9 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -245,7 +245,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB // Check if all used blobs have been found in index if len(usedBlobs) != 0 { - Warnf("%v not found in the new index\n"+ + Warnf("%v not found in the index\n"+ "Data blobs seem to be missing, aborting prune to prevent further data loss!\n"+ "Please report this error (along with the output of the 'prune' run) at\n"+ "https://github.com/restic/restic/issues/new/choose", usedBlobs)