\babel@toc {english}{}
\contentsline {chapter}{\numberline {1}S3QL}{1}{chapter.1}% 
\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}% 
\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}% 
\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}% 
\contentsline {section}{\numberline {1.4}Contributing}{2}{section.1.4}% 
\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}% 
\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}% 
\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}% 
\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}% 
\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}% 
\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}% 
\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}% 
\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}% 
\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}% 
\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}% 
\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}% 
\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}% 
\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}% 
\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}% 
\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}% 
\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}% 
\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}% 
\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}% 
\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}% 
\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}% 
\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}% 
\contentsline {section}{\numberline {6.3}Deleting a file system}{20}{section.6.3}% 
\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{20}{section.6.4}% 
\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7}% 
\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}% 
\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}% 
\contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3}% 
\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1}% 
\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}% 
\contentsline {section}{\numberline {7.4}NFS Support}{23}{section.7.4}% 
\contentsline {section}{\numberline {7.5}Failure Modes}{23}{section.7.5}% 
\contentsline {section}{\numberline {7.6}Automatic Mounting}{24}{section.7.6}% 
\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}% 
\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}% 
\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}% 
\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}% 
\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}% 
\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}% 
\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}% 
\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}% 
\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}% 
\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}% 
\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}% 
\contentsline {chapter}{\numberline {11}Storing Backend Information and Credentials}{35}{chapter.11}% 
\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}% 
\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}% 
\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}% 
\contentsline {section}{\numberline {12.3}pcp.py}{37}{section.12.3}% 
\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{37}{section.12.4}% 
\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5}% 
\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6}% 
\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13}% 
\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}% 
\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}% 
\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}% 
\contentsline {chapter}{\numberline {14}Frequently Asked Questions}{43}{chapter.14}% 
\contentsline {section}{\numberline {14.1}What does \IeC {\textquotedblleft }python-apsw must be linked dynamically to sqlite3\IeC {\textquotedblright } mean?}{43}{section.14.1}% 
\contentsline {section}{\numberline {14.2}How can I improve the file system throughput?}{43}{section.14.2}% 
\contentsline {section}{\numberline {14.3}Why does \sphinxstyleliteralintitle {\sphinxupquote {df}} show 1 TB of free space?}{43}{section.14.3}% 
\contentsline {section}{\numberline {14.4}Which operating systems are supported?}{44}{section.14.4}% 
\contentsline {section}{\numberline {14.5}Is there a file size limit?}{44}{section.14.5}% 
\contentsline {section}{\numberline {14.6}Suppose I want to make a small change in a very large file. Will S3QL download and re-upload the entire file?}{44}{section.14.6}% 
\contentsline {section}{\numberline {14.7}I don\IeC {\textquoteright }t quite understand this de-duplication feature\IeC {\textellipsis }}{44}{section.14.7}% 
\contentsline {section}{\numberline {14.8}What does the \IeC {\textquotedblleft }Transport endpoint not connected\IeC {\textquotedblright } error mean?}{44}{section.14.8}% 
\contentsline {section}{\numberline {14.9}What does \IeC {\textquotedblleft }Backend reports that fs is still mounted elsewhere, aborting\IeC {\textquotedblright } mean?}{44}{section.14.9}% 
\contentsline {section}{\numberline {14.10}Can I access an S3QL file system on multiple computers simultaneously?}{45}{section.14.10}% 
\contentsline {section}{\numberline {14.11}What maximum object size should I use?}{45}{section.14.11}% 
\contentsline {section}{\numberline {14.12}Is there a way to make the cache persistent / access the file system offline?}{46}{section.14.12}% 
\contentsline {section}{\numberline {14.13}I would like to use S3QL with Hubic, but\IeC {\textellipsis }}{46}{section.14.13}% 
\contentsline {section}{\numberline {14.14}What\IeC {\textquoteright }s a reasonable metadata upload interval?}{46}{section.14.14}% 
\contentsline {chapter}{\numberline {15}Known Issues}{49}{chapter.15}% 
\contentsline {chapter}{\numberline {16}Manpages}{51}{chapter.16}% 
\contentsline {section}{\numberline {16.1}The \sphinxstyleliteralstrong {\sphinxupquote {mkfs.s3ql}} command}{51}{section.16.1}% 
\contentsline {subsection}{\numberline {16.1.1}Synopsis}{51}{subsection.16.1.1}% 
\contentsline {subsection}{\numberline {16.1.2}Description}{51}{subsection.16.1.2}% 
\contentsline {subsection}{\numberline {16.1.3}Options}{51}{subsection.16.1.3}% 
\contentsline {subsection}{\numberline {16.1.4}Exit Codes}{52}{subsection.16.1.4}% 
\contentsline {subsection}{\numberline {16.1.5}See Also}{52}{subsection.16.1.5}% 
\contentsline {section}{\numberline {16.2}The \sphinxstyleliteralstrong {\sphinxupquote {s3qladm}} command}{52}{section.16.2}% 
\contentsline {subsection}{\numberline {16.2.1}Synopsis}{52}{subsection.16.2.1}% 
\contentsline {subsection}{\numberline {16.2.2}Description}{53}{subsection.16.2.2}% 
\contentsline {subsection}{\numberline {16.2.3}Options}{53}{subsection.16.2.3}% 
\contentsline {subsection}{\numberline {16.2.4}Actions}{53}{subsection.16.2.4}% 
\contentsline {subsection}{\numberline {16.2.5}Exit Codes}{53}{subsection.16.2.5}% 
\contentsline {subsection}{\numberline {16.2.6}See Also}{54}{subsection.16.2.6}% 
\contentsline {section}{\numberline {16.3}The \sphinxstyleliteralstrong {\sphinxupquote {mount.s3ql}} command}{54}{section.16.3}% 
\contentsline {subsection}{\numberline {16.3.1}Synopsis}{54}{subsection.16.3.1}% 
\contentsline {subsection}{\numberline {16.3.2}Description}{54}{subsection.16.3.2}% 
\contentsline {subsection}{\numberline {16.3.3}Options}{54}{subsection.16.3.3}% 
\contentsline {subsection}{\numberline {16.3.4}Exit Codes}{55}{subsection.16.3.4}% 
\contentsline {subsection}{\numberline {16.3.5}See Also}{56}{subsection.16.3.5}% 
\contentsline {section}{\numberline {16.4}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlstat}} command}{56}{section.16.4}% 
\contentsline {subsection}{\numberline {16.4.1}Synopsis}{56}{subsection.16.4.1}% 
\contentsline {subsection}{\numberline {16.4.2}Description}{57}{subsection.16.4.2}% 
\contentsline {subsection}{\numberline {16.4.3}Options}{57}{subsection.16.4.3}% 
\contentsline {subsection}{\numberline {16.4.4}Exit Codes}{57}{subsection.16.4.4}% 
\contentsline {subsection}{\numberline {16.4.5}See Also}{57}{subsection.16.4.5}% 
\contentsline {section}{\numberline {16.5}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlctrl}} command}{57}{section.16.5}% 
\contentsline {subsection}{\numberline {16.5.1}Synopsis}{57}{subsection.16.5.1}% 
\contentsline {subsection}{\numberline {16.5.2}Description}{58}{subsection.16.5.2}% 
\contentsline {subsection}{\numberline {16.5.3}Options}{58}{subsection.16.5.3}% 
\contentsline {subsection}{\numberline {16.5.4}Exit Codes}{58}{subsection.16.5.4}% 
\contentsline {subsection}{\numberline {16.5.5}See Also}{59}{subsection.16.5.5}% 
\contentsline {section}{\numberline {16.6}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlcp}} command}{59}{section.16.6}% 
\contentsline {subsection}{\numberline {16.6.1}Synopsis}{59}{subsection.16.6.1}% 
\contentsline {subsection}{\numberline {16.6.2}Description}{59}{subsection.16.6.2}% 
\contentsline {subsubsection}{Snapshotting vs Hardlinking}{59}{subsubsection*.28}% 
\contentsline {subsection}{\numberline {16.6.3}Options}{60}{subsection.16.6.3}% 
\contentsline {subsection}{\numberline {16.6.4}Exit Codes}{60}{subsection.16.6.4}% 
\contentsline {subsection}{\numberline {16.6.5}See Also}{60}{subsection.16.6.5}% 
\contentsline {section}{\numberline {16.7}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlrm}} command}{60}{section.16.7}% 
\contentsline {subsection}{\numberline {16.7.1}Synopsis}{60}{subsection.16.7.1}% 
\contentsline {subsection}{\numberline {16.7.2}Description}{60}{subsection.16.7.2}% 
\contentsline {subsection}{\numberline {16.7.3}Options}{61}{subsection.16.7.3}% 
\contentsline {subsection}{\numberline {16.7.4}Exit Codes}{61}{subsection.16.7.4}% 
\contentsline {subsection}{\numberline {16.7.5}See Also}{61}{subsection.16.7.5}% 
\contentsline {section}{\numberline {16.8}The \sphinxstyleliteralstrong {\sphinxupquote {s3qllock}} command}{61}{section.16.8}% 
\contentsline {subsection}{\numberline {16.8.1}Synopsis}{61}{subsection.16.8.1}% 
\contentsline {subsection}{\numberline {16.8.2}Description}{61}{subsection.16.8.2}% 
\contentsline {subsection}{\numberline {16.8.3}Rationale}{62}{subsection.16.8.3}% 
\contentsline {subsection}{\numberline {16.8.4}Options}{62}{subsection.16.8.4}% 
\contentsline {subsection}{\numberline {16.8.5}Exit Codes}{62}{subsection.16.8.5}% 
\contentsline {subsection}{\numberline {16.8.6}See Also}{62}{subsection.16.8.6}% 
\contentsline {section}{\numberline {16.9}The \sphinxstyleliteralstrong {\sphinxupquote {umount.s3ql}} command}{63}{section.16.9}% 
\contentsline {subsection}{\numberline {16.9.1}Synopsis}{63}{subsection.16.9.1}% 
\contentsline {subsection}{\numberline {16.9.2}Description}{63}{subsection.16.9.2}% 
\contentsline {subsection}{\numberline {16.9.3}Options}{63}{subsection.16.9.3}% 
\contentsline {subsection}{\numberline {16.9.4}Exit Codes}{63}{subsection.16.9.4}% 
\contentsline {subsection}{\numberline {16.9.5}See Also}{64}{subsection.16.9.5}% 
\contentsline {section}{\numberline {16.10}The \sphinxstyleliteralstrong {\sphinxupquote {fsck.s3ql}} command}{64}{section.16.10}% 
\contentsline {subsection}{\numberline {16.10.1}Synopsis}{64}{subsection.16.10.1}% 
\contentsline {subsection}{\numberline {16.10.2}Description}{64}{subsection.16.10.2}% 
\contentsline {subsection}{\numberline {16.10.3}Options}{64}{subsection.16.10.3}% 
\contentsline {subsection}{\numberline {16.10.4}Exit Codes}{65}{subsection.16.10.4}% 
\contentsline {subsection}{\numberline {16.10.5}See Also}{65}{subsection.16.10.5}% 
\contentsline {section}{\numberline {16.11}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_oauth\_client}} command}{66}{section.16.11}% 
\contentsline {subsection}{\numberline {16.11.1}Synopsis}{66}{subsection.16.11.1}% 
\contentsline {subsection}{\numberline {16.11.2}Description}{66}{subsection.16.11.2}% 
\contentsline {subsection}{\numberline {16.11.3}Options}{66}{subsection.16.11.3}% 
\contentsline {subsection}{\numberline {16.11.4}Exit Codes}{66}{subsection.16.11.4}% 
\contentsline {subsection}{\numberline {16.11.5}See Also}{66}{subsection.16.11.5}% 
\contentsline {section}{\numberline {16.12}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_verify}} command}{66}{section.16.12}% 
\contentsline {subsection}{\numberline {16.12.1}Synopsis}{66}{subsection.16.12.1}% 
\contentsline {subsection}{\numberline {16.12.2}Description}{67}{subsection.16.12.2}% 
\contentsline {subsection}{\numberline {16.12.3}Options}{67}{subsection.16.12.3}% 
\contentsline {subsection}{\numberline {16.12.4}Exit Codes}{67}{subsection.16.12.4}% 
\contentsline {subsection}{\numberline {16.12.5}See Also}{68}{subsection.16.12.5}% 
\contentsline {section}{\numberline {16.13}The \sphinxstyleliteralstrong {\sphinxupquote {pcp}} command}{68}{section.16.13}% 
\contentsline {subsection}{\numberline {16.13.1}Synopsis}{68}{subsection.16.13.1}% 
\contentsline {subsection}{\numberline {16.13.2}Description}{68}{subsection.16.13.2}% 
\contentsline {subsection}{\numberline {16.13.3}Options}{68}{subsection.16.13.3}% 
\contentsline {subsection}{\numberline {16.13.4}Exit Codes}{69}{subsection.16.13.4}% 
\contentsline {subsection}{\numberline {16.13.5}See Also}{69}{subsection.16.13.5}% 
\contentsline {section}{\numberline {16.14}The \sphinxstyleliteralstrong {\sphinxupquote {expire\_backups}} command}{69}{section.16.14}% 
\contentsline {subsection}{\numberline {16.14.1}Synopsis}{69}{subsection.16.14.1}% 
\contentsline {subsection}{\numberline {16.14.2}Description}{69}{subsection.16.14.2}% 
\contentsline {subsection}{\numberline {16.14.3}Options}{70}{subsection.16.14.3}% 
\contentsline {subsection}{\numberline {16.14.4}Exit Codes}{70}{subsection.16.14.4}% 
\contentsline {subsection}{\numberline {16.14.5}See Also}{71}{subsection.16.14.5}% 
\contentsline {chapter}{\numberline {17}Further Resources / Getting Help}{73}{chapter.17}% 
\contentsline {chapter}{\numberline {18}Implementation Details}{75}{chapter.18}% 
\contentsline {section}{\numberline {18.1}Metadata Storage}{75}{section.18.1}% 
\contentsline {section}{\numberline {18.2}Data Storage}{75}{section.18.2}% 
\contentsline {section}{\numberline {18.3}Data De-Duplication}{76}{section.18.3}% 
\contentsline {section}{\numberline {18.4}Caching}{76}{section.18.4}% 
\contentsline {section}{\numberline {18.5}Eventual Consistency Handling}{76}{section.18.5}% 
\contentsline {section}{\numberline {18.6}Encryption}{76}{section.18.6}% 
