From 4e2ff47b1c9ec3b72caef61ba2b05782406aeadb Mon Sep 17 00:00:00 2001 From: El RIDO Date: Sun, 13 Nov 2022 11:18:56 +0100 Subject: [PATCH 1/4] add an administrative script --- bin/privatebin-admin | 184 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 bin/privatebin-admin diff --git a/bin/privatebin-admin b/bin/privatebin-admin new file mode 100755 index 0000000..85d7574 --- /dev/null +++ b/bin/privatebin-admin @@ -0,0 +1,184 @@ +#!/usr/bin/env php + | --empty-dirs | --help | --statistics] + +Options: + -d, --delete deletes the requested paste ID + -e, --empty-dirs removes empty directories (only if Filesystem storage is + configured) + -h, --help displays this help message + -s, --statistics reads all stored pastes and comments and reports statistics +EOT, PHP_EOL; + exit($code); +} + +function option($short, $long) { + global $options; + $option = null; + foreach (array($short, $long) as $key) { + if (array_key_exists($key, $options)) { + $option = $options[$key]; + } + } + return $option; +} + +function main() { + if ($_SERVER['argc'] > 3) { + error_echo('too many arguments given'); + fwrite(STDERR, PHP_EOL); + help(1); + } + + if ($_SERVER['argc'] < 2) { + error_echo('missing arguments'); + fwrite(STDERR, PHP_EOL); + help(2); + } + + global $options; + $options = getopt('hd:eps', array('help', 'delete:', 'empty-dirs', 'purge', 'statistics')); + if (!$options) { + error_echo('unsupported arguments given'); + fwrite(STDERR, PHP_EOL); + help(3); + } + + if (option('h', 'help') !== null) { + help(); + } + + $conf = new Configuration; + + if (option('e', 'empty-dirs') !== null) { + if ($conf->getKey('class', 'model') !== 'Filesystem') { + error('instance not using Filesystem storage, no directories to empty', 4); + } + $dir = $conf->getKey('dir', 'model_options'); + passthru("find $dir -type d -empty -delete", $code); + exit($code); + } + + $class = 'PrivateBin\\Data\\' . $conf->getKey('class', 'model'); + $store = new $class($conf->getSection('model_options')); + + if (($pasteid = option('d', 'delete')) !== null) { + if (!Paste::isValidId($pasteid)) { + error('given ID is not a valid paste ID (16 hexadecimal digits)', 5); + } + if (!$store->exists($pasteid)) { + error('given ID does not exist, has expired or was already deleted', 6); + } + $store->delete($pasteid); + if ($store->exists($pasteid)) { + error('paste ID exists after deletion, permission problem?', 7); + } + exit("paste $pasteid successfully deleted" . PHP_EOL); + } + + if (option('p', 'purge') !== null) { + $store->purge(PHP_INT_MAX); + exit('purging of expired pastes concluded' . PHP_EOL); + } + + if (option('s', 'statistics') !== null) { + $counters = array( + 'burn' => 0, + 'discussion' => 0, + 'expired' => 0, + 'md' => 0, + 'percent' => 1, + 'plain' => 0, + 'progress' => 0, + 'syntax' => 0, + 'total' => 0, + 'unknown' => 0, + ); + $time = time(); + $ids = $store->getAllPastes(); + $counters['total'] = count($ids); + $dots = $counters['total'] < 100 ? 10 : ( + $counters['total'] < 1000 ? 50 : 100 + ); + $percentages = $counters['total'] < 100 ? 0 : ( + $counters['total'] < 1000 ? 4 : 10 + ); + echo "Total:\t\t\t${counters['total']}", PHP_EOL; + foreach ($ids as $pasteid) { + $paste = $store->read($pasteid); + ++$counters['progress']; + if ( + array_key_exists('expire_date', $paste['meta']) && + $paste['meta']['expire_date'] < $time + ) { + ++$counters['expired']; + } + if (array_key_exists('adata', $paste)) { + $format = $paste['adata'][1]; + $discussion = $paste['adata'][2]; + $burn = $paste['adata'][3]; + } else { + $format = array_key_exists('formatter', $paste['meta']) ? $paste['meta']['formatter'] : 'plaintext'; + $discussion = array_key_exists('opendiscussion', $paste['meta']) ? $paste['meta']['opendiscussion'] : false; + $burn = array_key_exists('burnafterreading', $paste['meta']) ? $paste['meta']['burnafterreading'] : false; + } + if ($format === 'plaintext') { + ++$counters['plain']; + } elseif ($format === 'syntaxhighlighting') { + ++$counters['syntax']; + } elseif ($format === 'markdown') { + ++$counters['md']; + } else { + ++$counters['unknown']; + } + $counters['discussion'] += (int) $discussion; + $counters['burn'] += (int) $burn; + + // display progress + if ($counters['progress'] % $dots === 0) { + echo '.'; + if ($percentages) { + $progress = $percentages / $counters['total'] * $counters['progress']; + if ($progress >= $counters['percent']) { + printf(' %d%% ', 100 / $percentages * $progress); + ++$counters['percent']; + } + } + } + } + echo PHP_EOL, << 0) { + echo "Unknown format:\t\t${counters['unknown']}", PHP_EOL; + } + } +} + +main(); \ No newline at end of file From 08854db6d60b42d8eeed95ce2711694f06993e85 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Sun, 13 Nov 2022 14:27:11 +0100 Subject: [PATCH 2/4] documented change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index edd8502..b4f1db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * **1.4.1 (not yet released)** * ADDED: script for data storage backend migrations (#1012) + * ADDED: script for administrative tasks: deleting pastes (#274), removing empty directories (#277), purging expired pastes (#276) & statistics (#319) * ADDED: Translations for Turkish, Slovak, Greek and Thai * ADDED: S3 Storage backend (#994) * ADDED: Jdenticons as an option for comment icons (#793) From 5c43ab6ef83d9d988f9fa1af0fd44cd9fa43b8d1 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Mon, 19 Dec 2022 20:41:12 +0100 Subject: [PATCH 3/4] refactor administrative script into OOP style and to our code guidelines --- CHANGELOG.md | 2 +- bin/administration | 317 +++++++++++++++++++++++++++++++++++++++++++ bin/privatebin-admin | 184 ------------------------- 3 files changed, 318 insertions(+), 185 deletions(-) create mode 100755 bin/administration delete mode 100755 bin/privatebin-admin diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bcce60..143e773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,12 @@ # PrivateBin version history * **1.5.1 (not yet released)** + * ADDED: script for administrative tasks: deleting pastes (#274), removing empty directories (#277), purging expired pastes (#276) & statistics (#319) * FIXED: Revert Filesystem purge to limited and randomized lookup (#1030) * FIXED: Catch JSON decode errors when invalid data gets sent to the API (#1030) * FIXED: Support sorting v1 format in mixed version comments in Filesystem backend (#1030) * **1.5 (2022-12-11)** * ADDED: script for data storage backend migrations (#1012) - * ADDED: script for administrative tasks: deleting pastes (#274), removing empty directories (#277), purging expired pastes (#276) & statistics (#319) * ADDED: Translations for Turkish, Slovak, Greek and Thai * ADDED: S3 Storage backend (#994) * ADDED: Jdenticons as an option for comment icons (#793) diff --git a/bin/administration b/bin/administration new file mode 100755 index 0000000..d3a7d63 --- /dev/null +++ b/bin/administration @@ -0,0 +1,317 @@ +#!/usr/bin/env php +_store->exists($pasteId)) { + self::_error('given ID does not exist, has expired or was already deleted', 6); + } + $this->_store->delete($pasteId); + if ($this->_store->exists($pasteId)) { + self::_error('paste ID exists after deletion, permission problem?', 7); + } + exit("paste $pasteId successfully deleted" . PHP_EOL); + } + + /** + * removes empty directories, if current storage model uses Filesystem + * + * @access private + */ + private function _empty_dirs() + { + if ($this->_conf->getKey('class', 'model') !== 'Filesystem') { + self::_error('instance not using Filesystem storage, no directories to empty', 4); + } + $dir = $this->_conf->getKey('dir', 'model_options'); + passthru("find $dir -type d -empty -delete", $code); + exit($code); + } + + /** + * display a message on STDERR and exits + * + * @access private + * @static + * @param string $message + * @param int $code optional, defaults to 1 + */ + private static function _error($message, $code = 1) + { + self::_error_echo($message); + exit($code); + } + + /** + * display a message on STDERR + * + * @access private + * @static + * @param string $message + */ + private static function _error_echo($message) + { + fwrite(STDERR, 'Error: ' . $message . PHP_EOL); + } + + /** + * display usage help on STDOUT and exits + * + * @access private + * @static + * @param int $code optional, defaults to 0 + */ + private static function _help($code = 0) + { + echo <<<'EOT' +Usage: + administration [--delete | --empty-dirs | --help | --statistics] + +Options: + -d, --delete deletes the requested paste ID + -e, --empty-dirs removes empty directories (only if Filesystem storage is + configured) + -h, --help displays this help message + -s, --statistics reads all stored pastes and comments and reports statistics +EOT, PHP_EOL; + exit($code); + } + + /** + * return option for given short or long keyname, if it got set + * + * @access private + * @static + * @param string $short + * @param string $long + * @return string|null + */ + private function _option($short, $long) + { + foreach (array($short, $long) as $key) { + if (array_key_exists($key, $this->_opts)) { + return $this->_opts[$key]; + } + } + return null; + } + + /** + * initialize options from given argument array + * + * @access private + * @static + * @param array $arguments + */ + private function _options_initialize($arguments) + { + if ($arguments > 3) { + self::_error_echo('too many arguments given'); + echo PHP_EOL; + self::_help(1); + } + + if ($arguments < 2) { + self::_error_echo('missing arguments'); + echo PHP_EOL; + self::_help(2); + } + + $this->_opts = getopt('hd:eps', array('help', 'delete:', 'empty-dirs', 'purge', 'statistics')); + if (!$this->_opts) { + self::_error_echo('unsupported arguments given'); + echo PHP_EOL; + self::_help(3); + } + } + + /** + * reads all stored pastes and comments and reports statistics + * + * @access public + */ + private function _statistics() + { + $counters = array( + 'burn' => 0, + 'discussion' => 0, + 'expired' => 0, + 'md' => 0, + 'percent' => 1, + 'plain' => 0, + 'progress' => 0, + 'syntax' => 0, + 'total' => 0, + 'unknown' => 0, + ); + $time = time(); + $ids = $this->_store->getAllPastes(); + $counters['total'] = count($ids); + $dots = $counters['total'] < 100 ? 10 : ( + $counters['total'] < 1000 ? 50 : 100 + ); + $percentages = $counters['total'] < 100 ? 0 : ( + $counters['total'] < 1000 ? 4 : 10 + ); + + echo "Total:\t\t\t${counters['total']}", PHP_EOL; + foreach ($ids as $pasteid) { + $paste = $this->_store->read($pasteid); + ++$counters['progress']; + + if ( + array_key_exists('expire_date', $paste['meta']) && + $paste['meta']['expire_date'] < $time + ) { + ++$counters['expired']; + } + + if (array_key_exists('adata', $paste)) { + $format = $paste['adata'][1]; + $discussion = $paste['adata'][2]; + $burn = $paste['adata'][3]; + } else { + $format = array_key_exists('formatter', $paste['meta']) ? $paste['meta']['formatter'] : 'plaintext'; + $discussion = array_key_exists('opendiscussion', $paste['meta']) ? $paste['meta']['opendiscussion'] : false; + $burn = array_key_exists('burnafterreading', $paste['meta']) ? $paste['meta']['burnafterreading'] : false; + } + + if ($format === 'plaintext') { + ++$counters['plain']; + } elseif ($format === 'syntaxhighlighting') { + ++$counters['syntax']; + } elseif ($format === 'markdown') { + ++$counters['md']; + } else { + ++$counters['unknown']; + } + + $counters['discussion'] += (int) $discussion; + $counters['burn'] += (int) $burn; + + // display progress + if ($counters['progress'] % $dots === 0) { + echo '.'; + if ($percentages) { + $progress = $percentages / $counters['total'] * $counters['progress']; + if ($progress >= $counters['percent']) { + printf(' %d%% ', 100 / $percentages * $progress); + ++$counters['percent']; + } + } + } + } + + echo PHP_EOL, << 0) { + echo "Unknown format:\t\t${counters['unknown']}", PHP_EOL; + } + } + + /** + * constructor + * + * initializes and runs administrative tasks + * + * @access public + */ + public function __construct() + { + $this->_options_initialize($_SERVER['argc']); + + if ($this->_option('h', 'help') !== null) { + self::_help(); + } + + $this->_conf = new Configuration; + + if ($this->_option('e', 'empty-dirs') !== null) { + $this->_empty_dirs(); + } + + $class = 'PrivateBin\\Data\\' . $this->_conf->getKey('class', 'model'); + $this->_store = new $class($this->_conf->getSection('model_options')); + + if (($pasteId = $this->_option('d', 'delete')) !== null) { + $this->_delete($pasteId); + } + + if ($this->_option('p', 'purge') !== null) { + $this->_store->purge(PHP_INT_MAX); + exit('purging of expired pastes concluded' . PHP_EOL); + } + + if ($this->_option('s', 'statistics') !== null) { + $this->_statistics(); + } + } +} + +new Administration(); \ No newline at end of file diff --git a/bin/privatebin-admin b/bin/privatebin-admin deleted file mode 100755 index 85d7574..0000000 --- a/bin/privatebin-admin +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env php - | --empty-dirs | --help | --statistics] - -Options: - -d, --delete deletes the requested paste ID - -e, --empty-dirs removes empty directories (only if Filesystem storage is - configured) - -h, --help displays this help message - -s, --statistics reads all stored pastes and comments and reports statistics -EOT, PHP_EOL; - exit($code); -} - -function option($short, $long) { - global $options; - $option = null; - foreach (array($short, $long) as $key) { - if (array_key_exists($key, $options)) { - $option = $options[$key]; - } - } - return $option; -} - -function main() { - if ($_SERVER['argc'] > 3) { - error_echo('too many arguments given'); - fwrite(STDERR, PHP_EOL); - help(1); - } - - if ($_SERVER['argc'] < 2) { - error_echo('missing arguments'); - fwrite(STDERR, PHP_EOL); - help(2); - } - - global $options; - $options = getopt('hd:eps', array('help', 'delete:', 'empty-dirs', 'purge', 'statistics')); - if (!$options) { - error_echo('unsupported arguments given'); - fwrite(STDERR, PHP_EOL); - help(3); - } - - if (option('h', 'help') !== null) { - help(); - } - - $conf = new Configuration; - - if (option('e', 'empty-dirs') !== null) { - if ($conf->getKey('class', 'model') !== 'Filesystem') { - error('instance not using Filesystem storage, no directories to empty', 4); - } - $dir = $conf->getKey('dir', 'model_options'); - passthru("find $dir -type d -empty -delete", $code); - exit($code); - } - - $class = 'PrivateBin\\Data\\' . $conf->getKey('class', 'model'); - $store = new $class($conf->getSection('model_options')); - - if (($pasteid = option('d', 'delete')) !== null) { - if (!Paste::isValidId($pasteid)) { - error('given ID is not a valid paste ID (16 hexadecimal digits)', 5); - } - if (!$store->exists($pasteid)) { - error('given ID does not exist, has expired or was already deleted', 6); - } - $store->delete($pasteid); - if ($store->exists($pasteid)) { - error('paste ID exists after deletion, permission problem?', 7); - } - exit("paste $pasteid successfully deleted" . PHP_EOL); - } - - if (option('p', 'purge') !== null) { - $store->purge(PHP_INT_MAX); - exit('purging of expired pastes concluded' . PHP_EOL); - } - - if (option('s', 'statistics') !== null) { - $counters = array( - 'burn' => 0, - 'discussion' => 0, - 'expired' => 0, - 'md' => 0, - 'percent' => 1, - 'plain' => 0, - 'progress' => 0, - 'syntax' => 0, - 'total' => 0, - 'unknown' => 0, - ); - $time = time(); - $ids = $store->getAllPastes(); - $counters['total'] = count($ids); - $dots = $counters['total'] < 100 ? 10 : ( - $counters['total'] < 1000 ? 50 : 100 - ); - $percentages = $counters['total'] < 100 ? 0 : ( - $counters['total'] < 1000 ? 4 : 10 - ); - echo "Total:\t\t\t${counters['total']}", PHP_EOL; - foreach ($ids as $pasteid) { - $paste = $store->read($pasteid); - ++$counters['progress']; - if ( - array_key_exists('expire_date', $paste['meta']) && - $paste['meta']['expire_date'] < $time - ) { - ++$counters['expired']; - } - if (array_key_exists('adata', $paste)) { - $format = $paste['adata'][1]; - $discussion = $paste['adata'][2]; - $burn = $paste['adata'][3]; - } else { - $format = array_key_exists('formatter', $paste['meta']) ? $paste['meta']['formatter'] : 'plaintext'; - $discussion = array_key_exists('opendiscussion', $paste['meta']) ? $paste['meta']['opendiscussion'] : false; - $burn = array_key_exists('burnafterreading', $paste['meta']) ? $paste['meta']['burnafterreading'] : false; - } - if ($format === 'plaintext') { - ++$counters['plain']; - } elseif ($format === 'syntaxhighlighting') { - ++$counters['syntax']; - } elseif ($format === 'markdown') { - ++$counters['md']; - } else { - ++$counters['unknown']; - } - $counters['discussion'] += (int) $discussion; - $counters['burn'] += (int) $burn; - - // display progress - if ($counters['progress'] % $dots === 0) { - echo '.'; - if ($percentages) { - $progress = $percentages / $counters['total'] * $counters['progress']; - if ($progress >= $counters['percent']) { - printf(' %d%% ', 100 / $percentages * $progress); - ++$counters['percent']; - } - } - } - } - echo PHP_EOL, << 0) { - echo "Unknown format:\t\t${counters['unknown']}", PHP_EOL; - } - } -} - -main(); \ No newline at end of file From 5de779a989a103bdd092b6947c014d19af8a5b54 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Tue, 20 Dec 2022 19:14:37 +0100 Subject: [PATCH 4/4] improve documentation --- bin/administration | 3 ++- bin/migrate | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/administration b/bin/administration index d3a7d63..bc82b43 100755 --- a/bin/administration +++ b/bin/administration @@ -125,13 +125,14 @@ class Administration { echo <<<'EOT' Usage: - administration [--delete | --empty-dirs | --help | --statistics] + administration [--delete | --empty-dirs | --help | --purge | --statistics] Options: -d, --delete deletes the requested paste ID -e, --empty-dirs removes empty directories (only if Filesystem storage is configured) -h, --help displays this help message + -p, --purge purge all expired pastes -s, --statistics reads all stored pastes and comments and reports statistics EOT, PHP_EOL; exit($code); diff --git a/bin/migrate b/bin/migrate index 515a5e8..76539ab 100755 --- a/bin/migrate +++ b/bin/migrate @@ -17,13 +17,14 @@ if (version_compare(PHP_VERSION, '7.1.0') < 0) { $longopts = array( "delete-after", - "delete-during" + "delete-during", + "help" ); $opts_arr = getopt("fhnv", $longopts, $rest); if ($opts_arr === false) { - dieerr("Erroneous command line options. Please use -h"); + dieerr("Erroneous command line options. Please use --help"); } -if (array_key_exists("h", $opts_arr)) { +if (array_key_exists("h", $opts_arr) || array_key_exists("help", $opts_arr)) { helpexit(); } @@ -173,12 +174,12 @@ function debug ($text) { function helpexit () { - print("migrate.php - Copy data between PrivateBin backends + print("migrate - Copy data between PrivateBin backends Usage: migrate [--delete-after] [--delete-during] [-f] [-n] [-v] srcconfdir [] - migrate [-h] + migrate [-h|--help] Options: --delete-after delete data from source after all pastes and comments have @@ -187,6 +188,7 @@ Options: comments have successfully been copied to the destination -f forcefully overwrite data which already exists at the destination + -h, --help displays this help message -n dry run, do not copy data -v be verbose use storage backend configration from conf.php found in