From 2182cdd44f092277ce829cb9effd861ceb415c06 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Sat, 22 Jan 2022 08:45:12 +0100 Subject: [PATCH] generalize OCI handling of queries and results --- CHANGELOG.md | 11 +- CREDITS.md | 1 + lib/Data/Database.php | 242 ++++++++++++++------------------------ tst/Data/DatabaseTest.php | 12 ++ 4 files changed, 105 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d964a57..428a60a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * ADDED: Opt-out of federated learning of cohorts (FLoC) (#776) * ADDED: Configuration option to exempt IPs from the rate-limiter (#787) * ADDED: Google Cloud Storage backend support (#795) + * ADDED: Oracle database support (#868) * CHANGED: Language selection cookie only transmitted over HTTPS (#472) * CHANGED: Upgrading libraries to: random_compat 2.0.20 * CHANGED: Removed automatic `.ini` configuration file migration (#808) @@ -65,7 +66,7 @@ * CHANGED: Upgrading libraries to: DOMpurify 2.0.1 * FIXED: Enabling browsers without WASM to create pastes and read uncompressed ones (#454) * FIXED: Cloning related issues (#489, #491, #493, #494) - * FIXED: Enable file operation only when editing (#497) + * FIXED: Enable file operation only when editing (#497) * FIXED: Clicking 'New' on a previously submitted paste does not blank address bar (#354) * FIXED: Clear address bar when create new paste from existing paste (#479) * FIXED: Discussion section not hiding when new/clone paste is clicked on (#484) @@ -228,7 +229,7 @@ encryption), i18n (translation, counterpart of i18n.php) and helper (stateless u * FIXED: 2 minor corrections to avoid notices in php log. * FIXED: Sources converted to UTF-8. * **Alpha 0.14 (2012-04-20):** - * ADDED: GD presence is checked. + * ADDED: GD presence is checked. * CHANGED: Traffic limiter data files moved to data/ (→easier rights management) * ADDED: "Burn after reading" implemented. Opening the URL will display the paste and immediately destroy it on server. * **Alpha 0.13 (2012-04-18):** @@ -236,16 +237,16 @@ encryption), i18n (translation, counterpart of i18n.php) and helper (stateless u * FIXED: $error not properly initialized in index.php * **Alpha 0.12 (2012-04-18):** * **DISCUSSIONS !** Now you can enable discussions on your pastes. Of course, posted comments and nickname are also encrypted and the server cannot see them. - * This feature implies a change in storage format. You will have to delete all previous pastes in your ZeroBin. + * This feature implies a change in storage format. You will have to delete all previous pastes in your ZeroBin. * Added [[php:vizhash_gd|Vizhash]] as avatars, so you can match posters IP addresses without revealing them. (Same image = same IP). Of course the IP address cannot be deduced from the Vizhash. * Remaining time before expiration is now displayed. - * Explicit tags were added to CSS and jQuery selectors (eg. div#aaa instead of #aaa) to speed up browser. + * Explicit tags were added to CSS and jQuery selectors (eg. div#aaa instead of #aaa) to speed up browser. * Better cleaning of the URL (to make sure the key is not broken by some stupid redirection service) * **Alpha 0.11 (2012-04-12):** * Automatically ignore parameters (such as &utm_source=...) added //after// the anchor by some stupid Web 2.0 services. * First public release. * **Alpha 0.10 (2012-04-12):** - * IE9 does not seem to correctly support ''pre-wrap'' either. Special handling mode activated for all version of IE<10. (Note: **ALL other browsers** correctly support this feature.) + * IE9 does not seem to correctly support ''pre-wrap'' either. Special handling mode activated for all version of IE<10. (Note: **ALL other browsers** correctly support this feature.) * **Alpha 0.9 (2012-04-11):** * Oh bummer... IE 8 is as shitty as IE6/7: Its does not seem to support ''white-space:pre-wrap'' correctly. I had to activate the special handling mode. I still have to test IE 9. * **Alpha 0.8 (2012-04-11):** diff --git a/CREDITS.md b/CREDITS.md index 612749c..6c2f647 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -29,6 +29,7 @@ Sébastien Sauvage - original idea and main developer * Lucas Savva - configurable config file location, NixOS packaging * rodehoed - option to exempt ips from the rate-limiter * Mark van Holsteijn - Google Cloud Storage backend +* Austin Huang - Oracle database support ## Translations * Hexalyse - French diff --git a/lib/Data/Database.php b/lib/Data/Database.php index a89da4b..f616a96 100644 --- a/lib/Data/Database.php +++ b/lib/Data/Database.php @@ -199,31 +199,12 @@ class Database extends AbstractData $burnafterreading = $paste['adata'][3]; } try { - $big_string = $isVersion1 ? $paste['data'] : Json::encode($paste); - $metajson = Json::encode($meta); - if (self::$_type === 'oci') { - // It is not possible to execute in the normal way if strlen($big_string) >= 4000 - $stmt = self::$_db->prepare( - 'INSERT INTO ' . self::_sanitizeIdentifier('paste') . - ' VALUES(?,?,?,?,?,?,?,?,?)' - ); - $stmt->bindParam(1, $pasteid); - $stmt->bindParam(2, $big_string, PDO::PARAM_STR, strlen($big_string)); - $stmt->bindParam(3, $created, PDO::PARAM_INT); - $stmt->bindParam(4, $expire_date, PDO::PARAM_INT); - $stmt->bindParam(5, $opendiscussion, PDO::PARAM_INT); - $stmt->bindParam(6, $burnafterreading, PDO::PARAM_INT); - $stmt->bindParam(7, $metajson); - $stmt->bindParam(8, $attachment, PDO::PARAM_STR, strlen($attachment)); - $stmt->bindParam(9, $attachmentname); - return $stmt->execute(); - } return self::_exec( 'INSERT INTO ' . self::_sanitizeIdentifier('paste') . ' VALUES(?,?,?,?,?,?,?,?,?)', array( $pasteid, - $big_string, + $isVersion1 ? $paste['data'] : Json::encode($paste), $created, $expire_date, (int) $opendiscussion, @@ -252,15 +233,11 @@ class Database extends AbstractData } self::$_cache[$pasteid] = false; - $rawData = ''; try { $paste = self::_select( 'SELECT * FROM ' . self::_sanitizeIdentifier('paste') . ' WHERE dataid = ?', array($pasteid), true ); - if ($paste !== false) { - $rawData = self::$_type === 'oci' ? self::_clob($paste['DATA']) : $paste['data']; - } } catch (Exception $e) { $paste = false; } @@ -268,25 +245,25 @@ class Database extends AbstractData return false; } // create array - $data = Json::decode($rawData); + $data = Json::decode($paste['data']); $isVersion2 = array_key_exists('v', $data) && $data['v'] >= 2; if ($isVersion2) { self::$_cache[$pasteid] = $data; list($createdKey) = self::_getVersionedKeys(2); } else { - self::$_cache[$pasteid] = array('data' => $paste[self::_sanitizeColumn('data')]); + self::$_cache[$pasteid] = array('data' => $paste['data']); list($createdKey) = self::_getVersionedKeys(1); } try { - $paste['meta'] = Json::decode($paste[self::_sanitizeColumn('meta')]); + $paste['meta'] = Json::decode($paste['meta']); } catch (Exception $e) { $paste['meta'] = array(); } $paste = self::upgradePreV1Format($paste); self::$_cache[$pasteid]['meta'] = $paste['meta']; - self::$_cache[$pasteid]['meta'][$createdKey] = (int) $paste[self::_sanitizeColumn('postdate')]; - $expire_date = (int) $paste[self::_sanitizeColumn('expiredate')]; + self::$_cache[$pasteid]['meta'][$createdKey] = (int) $paste['postdate']; + $expire_date = (int) $paste['expiredate']; if ($expire_date > 0) { self::$_cache[$pasteid]['meta']['expire_date'] = $expire_date; } @@ -295,16 +272,16 @@ class Database extends AbstractData } // support v1 attachments - if (array_key_exists(self::_sanitizeColumn('attachment'), $paste) && strlen($paste[self::_sanitizeColumn('attachment')])) { - self::$_cache[$pasteid]['attachment'] = $paste[self::_sanitizeColumn('attachment')]; - if (array_key_exists(self::_sanitizeColumn('attachmentname'), $paste) && strlen($paste[self::_sanitizeColumn('attachmentname')])) { - self::$_cache[$pasteid]['attachmentname'] = $paste[self::_sanitizeColumn('attachmentname')]; + if (array_key_exists('attachment', $paste) && strlen($paste['attachment'])) { + self::$_cache[$pasteid]['attachment'] = $paste['attachment']; + if (array_key_exists('attachmentname', $paste) && strlen($paste['attachmentname'])) { + self::$_cache[$pasteid]['attachmentname'] = $paste['attachmentname']; } } - if ($paste[self::_sanitizeColumn('opendiscussion')]) { + if ($paste['opendiscussion']) { self::$_cache[$pasteid]['meta']['opendiscussion'] = true; } - if ($paste[self::_sanitizeColumn('burnafterreading')]) { + if ($paste['burnafterreading']) { self::$_cache[$pasteid]['meta']['burnafterreading'] = true; } @@ -379,21 +356,6 @@ class Database extends AbstractData } } try { - if (self::$_type === 'oci') { - // It is not possible to execute in the normal way if strlen($big_string) >= 4000 - $stmt = self::$_db->prepare( - 'INSERT INTO ' . self::_sanitizeIdentifier('comment') . - ' VALUES(?,?,?,?,?,?,?)' - ); - $stmt->bindParam(1, $commentid); - $stmt->bindParam(2, $pasteid); - $stmt->bindParam(3, $parentid); - $stmt->bindParam(4, $data, PDO::PARAM_STR, strlen($data)); - $stmt->bindParam(5, $meta['nickname']); - $stmt->bindParam(6, $meta[$iconKey]); - $stmt->bindParam(7, $meta[$createdKey], PDO::PARAM_INT); - return $stmt->execute(); - } return self::_exec( 'INSERT INTO ' . self::_sanitizeIdentifier('comment') . ' VALUES(?,?,?,?,?,?,?)', @@ -430,32 +392,22 @@ class Database extends AbstractData $comments = array(); if (count($rows)) { foreach ($rows as $row) { - $i = $this->getOpenSlot($comments, (int) $row[self::_sanitizeColumn('postdate')]); - $id = $row[self::_sanitizeColumn('dataid')]; - if (self::$_type === 'oci') { - $newrow = self::_select( - 'SELECT data FROM ' . self::_sanitizeIdentifier('comment') . - ' WHERE dataid = ?', array($id), true - ); - $rawData = self::_clob($newrow['DATA']); - } else { - $rawData = $row['data']; - } - $data = Json::decode($rawData); + $i = $this->getOpenSlot($comments, (int) $row['postdate']); + $data = Json::decode($row['data']); if (array_key_exists('v', $data) && $data['v'] >= 2) { $version = 2; $comments[$i] = $data; } else { $version = 1; - $comments[$i] = array('data' => $rawData); + $comments[$i] = array('data' => $row['data']); } list($createdKey, $iconKey) = self::_getVersionedKeys($version); - $comments[$i]['id'] = $id; - $comments[$i]['parentid'] = $row[self::_sanitizeColumn('parentid')]; - $comments[$i]['meta'] = array($createdKey => (int) $row[self::_sanitizeColumn('postdate')]); + $comments[$i]['id'] = $row['dataid']; + $comments[$i]['parentid'] = $row['parentid']; + $comments[$i]['meta'] = array($createdKey => (int) $row['postdate']); foreach (array('nickname' => 'nickname', 'vizhash' => $iconKey) as $rowKey => $commentKey) { - if (array_key_exists(self::_sanitizeColumn($rowKey), $row) && !empty($row[self::_sanitizeColumn($rowKey)])) { - $comments[$i]['meta'][$commentKey] = $row[self::_sanitizeColumn($rowKey)]; + if (array_key_exists($rowKey, $row) && !empty($row[$rowKey])) { + $comments[$i]['meta'][$commentKey] = $row[$rowKey]; } } } @@ -572,7 +524,7 @@ class Database extends AbstractData ); if (count($rows)) { foreach ($rows as $row) { - $pastes[] = $row[self::_sanitizeIdentifier('dataid')]; + $pastes[] = $row['dataid']; } } return $pastes; @@ -591,7 +543,22 @@ class Database extends AbstractData private static function _exec($sql, array $params) { $statement = self::$_db->prepare($sql); - $result = $statement->execute($params); + if (self::$_type === 'oci') { + // It is not possible to execute in the normal way if strlen($param) >= 4000 + foreach ($params as $key => $parameter) { + $position = $key + 1; + if (is_int($parameter)) { + $statement->bindParam($position, $parameter, PDO::PARAM_INT); + } elseif ($length = strlen($parameter) >= 4000) { + $statement->bindParam($position, $parameter, PDO::PARAM_STR, $length); + } else { + $statement->bindParam($position, $parameter); + } + } + $result = $statement->execute(); + } else { + $result = $statement->execute($params); + } $statement->closeCursor(); return $result; } @@ -615,6 +582,14 @@ class Database extends AbstractData $statement->fetch(PDO::FETCH_ASSOC) : $statement->fetchAll(PDO::FETCH_ASSOC); $statement->closeCursor(); + if (self::$_type === 'oci') { + // returned column names are all upper case, convert these back + // returned CLOB values are streams, convert these into strings + $result = array_combine( + array_map('strtolower', array_keys($result)), + array_map('self::_sanitizeClob', array_values($result)) + ); + } return $result; } @@ -707,19 +682,7 @@ class Database extends AbstractData } catch (PDOException $e) { return ''; } - return $row ? $row[self::_sanitizeColumn('value')] : ''; - } - - /** - * OCI cannot accept semicolons - * - * @access private - * @static - * @return string - */ - private static function _getSemicolon() - { - return self::$_type === 'oci' ? '' : ';'; + return $row ? $row['value'] : ''; } /** @@ -744,7 +707,7 @@ class Database extends AbstractData /** * get the data type, depending on the database driver * - * PostgreSQL uses a different API for BLOBs then SQL, hence we use TEXT + * PostgreSQL and OCI uses a different API for BLOBs then SQL, hence we use TEXT and CLOB * * @access private * @static @@ -752,7 +715,7 @@ class Database extends AbstractData */ private static function _getDataType() { - return self::$_type === 'pgsql' ? 'TEXT' : (self::$_type === 'oci' ? 'VARCHAR2(4000)' : 'BLOB'); + return self::$_type === 'pgsql' ? 'TEXT' : (self::$_type === 'oci' ? 'CLOB' : 'BLOB'); } /** @@ -772,7 +735,7 @@ class Database extends AbstractData /** * get the meta type, depending on the database driver * - * OCI can't even accept TEXT so it has to be VARCHAR2(200) + * OCI doesn't accept TEXT so it has to be VARCHAR2(4000) * * @access private * @static @@ -798,31 +761,17 @@ class Database extends AbstractData self::$_db->exec( 'CREATE TABLE ' . self::_sanitizeIdentifier('paste') . ' ( ' . "dataid CHAR(16) NOT NULL$main_key, " . - "data $attachmentType, " . + "data $dataType, " . 'postdate INT, ' . 'expiredate INT, ' . 'opendiscussion INT, ' . 'burnafterreading INT, ' . "meta $metaType, " . "attachment $attachmentType, " . - "attachmentname $dataType$after_key )" . self::_getSemicolon() + "attachmentname $dataType$after_key )" ); } - /** - * get the nullable text type, depending on the database driver - * - * OCI will pad CHAR columns with spaces, hence VARCHAR2 - * - * @access private - * @static - * @return string - */ - private static function _getParentType() - { - return self::$_type === 'oci' ? 'VARCHAR2(16)' : 'CHAR(16)'; - } - /** * create the paste table * @@ -833,21 +782,19 @@ class Database extends AbstractData { list($main_key, $after_key) = self::_getPrimaryKeyClauses(); $dataType = self::_getDataType(); - $parentType = self::_getParentType(); - $attachmentType = self::_getAttachmentType(); self::$_db->exec( 'CREATE TABLE ' . self::_sanitizeIdentifier('comment') . ' ( ' . "dataid CHAR(16) NOT NULL$main_key, " . 'pasteid CHAR(16), ' . - "parentid $parentType, " . - "data $attachmentType, " . + 'parentid CHAR(16), ' . + "data $dataType, " . "nickname $dataType, " . "vizhash $dataType, " . - "postdate INT$after_key )" . self::_getSemicolon() + "postdate INT$after_key )" ); self::$_db->exec( - 'CREATE INDEX comment_parent ON ' . - self::_sanitizeIdentifier('comment') . '(pasteid)' . self::_getSemicolon() + 'CREATE INDEX IF NOT EXISTS comment_parent ON ' . + self::_sanitizeIdentifier('comment') . '(pasteid)' ); } @@ -864,7 +811,7 @@ class Database extends AbstractData $textType = self::$_type === 'oci' ? 'VARCHAR2(4000)' : 'TEXT'; self::$_db->exec( 'CREATE TABLE ' . self::_sanitizeIdentifier('config') . - " ( id $charType NOT NULL$main_key, value $textType$after_key )" . self::_getSemicolon() + " ( id $charType NOT NULL$main_key, value $textType$after_key )" ); self::_exec( 'INSERT INTO ' . self::_sanitizeIdentifier('config') . @@ -873,6 +820,24 @@ class Database extends AbstractData ); } + /** + * sanitizes CLOB values used with OCI + * + * From: https://stackoverflow.com/questions/36200534/pdo-oci-into-a-clob-field + * + * @access private + * @static + * @param int|string|resource $value + * @return int|string + */ + public static function _sanitizeClob($value) + { + if (is_resource($value)) { + $value = stream_get_contents($value); + } + return $value; + } + /** * sanitizes identifiers * @@ -883,21 +848,7 @@ class Database extends AbstractData */ private static function _sanitizeIdentifier($identifier) { - $id = preg_replace('/[^A-Za-z0-9_]+/', '', self::$_prefix . $identifier); - return self::_sanitizeColumn($id); - } - - /** - * sanitizes column name because OCI - * - * @access private - * @static - * @param string $name - * @return string - */ - private static function _sanitizeColumn($name) - { - return self::$_type === 'oci' ? strtoupper($name) : $name; + return preg_replace('/[^A-Za-z0-9_]+/', '', self::$_prefix . $identifier); } /** @@ -915,43 +866,43 @@ class Database extends AbstractData case '0.21': // create the meta column if necessary (pre 0.21 change) try { - self::$_db->exec('SELECT meta FROM ' . self::_sanitizeIdentifier('paste') . ' LIMIT 1;'); + self::$_db->exec('SELECT meta FROM ' . self::_sanitizeIdentifier('paste') . ' LIMIT 1'); } catch (PDOException $e) { - self::$_db->exec('ALTER TABLE ' . self::_sanitizeIdentifier('paste') . ' ADD COLUMN meta TEXT;'); + self::$_db->exec('ALTER TABLE ' . self::_sanitizeIdentifier('paste') . ' ADD COLUMN meta TEXT'); } // SQLite only allows one ALTER statement at a time... self::$_db->exec( 'ALTER TABLE ' . self::_sanitizeIdentifier('paste') . - " ADD COLUMN attachment $attachmentType;" + " ADD COLUMN attachment $attachmentType" ); self::$_db->exec( - 'ALTER TABLE ' . self::_sanitizeIdentifier('paste') . " ADD COLUMN attachmentname $dataType;" + 'ALTER TABLE ' . self::_sanitizeIdentifier('paste') . " ADD COLUMN attachmentname $dataType" ); // SQLite doesn't support MODIFY, but it allows TEXT of similar // size as BLOB, so there is no need to change it there if (self::$_type !== 'sqlite') { self::$_db->exec( 'ALTER TABLE ' . self::_sanitizeIdentifier('paste') . - " ADD PRIMARY KEY (dataid), MODIFY COLUMN data $dataType;" + " ADD PRIMARY KEY (dataid), MODIFY COLUMN data $dataType" ); self::$_db->exec( 'ALTER TABLE ' . self::_sanitizeIdentifier('comment') . " ADD PRIMARY KEY (dataid), MODIFY COLUMN data $dataType, " . - "MODIFY COLUMN nickname $dataType, MODIFY COLUMN vizhash $dataType;" + "MODIFY COLUMN nickname $dataType, MODIFY COLUMN vizhash $dataType" ); } else { self::$_db->exec( 'CREATE UNIQUE INDEX IF NOT EXISTS paste_dataid ON ' . - self::_sanitizeIdentifier('paste') . '(dataid);' + self::_sanitizeIdentifier('paste') . '(dataid)' ); self::$_db->exec( 'CREATE UNIQUE INDEX IF NOT EXISTS comment_dataid ON ' . - self::_sanitizeIdentifier('comment') . '(dataid);' + self::_sanitizeIdentifier('comment') . '(dataid)' ); } self::$_db->exec( 'CREATE INDEX IF NOT EXISTS comment_parent ON ' . - self::_sanitizeIdentifier('comment') . '(pasteid);' + self::_sanitizeIdentifier('comment') . '(pasteid)' ); // no break, continue with updates for 0.22 and later case '1.3': @@ -961,7 +912,7 @@ class Database extends AbstractData if (self::$_type !== 'sqlite' && self::$_type !== 'pgsql') { self::$_db->exec( 'ALTER TABLE ' . self::_sanitizeIdentifier('paste') . - " MODIFY COLUMN data $attachmentType;" + " MODIFY COLUMN data $attachmentType" ); } // no break, continue with updates for all newer versions @@ -973,25 +924,4 @@ class Database extends AbstractData ); } } - - /** - * read CLOB for OCI - * https://stackoverflow.com/questions/36200534/pdo-oci-into-a-clob-field - * - * @access private - * @static - * @param resource $column - * @return string - */ - private static function _clob($column) - { - if ($column == null) { - return null; - } - $str = ''; - while ($tmp = fread($column, 1024)) { - $str .= $tmp; - } - return $str; - } } diff --git a/tst/Data/DatabaseTest.php b/tst/Data/DatabaseTest.php index 1cfc0be..c433b5a 100644 --- a/tst/Data/DatabaseTest.php +++ b/tst/Data/DatabaseTest.php @@ -388,4 +388,16 @@ class DatabaseTest extends PHPUnit_Framework_TestCase $this->assertEquals(Controller::VERSION, $result['value']); Helper::rmDir($this->_path); } + + public function testOciClob() + { + $int = (int) random_bytes(1); + $string = random_bytes(10); + $clob = fopen('php://memory', 'r+'); + fwrite($clob, $string); + rewind($clob); + $this->assertEquals($int, Database::_sanitizeClob($int)); + $this->assertEquals($string, Database::_sanitizeClob($string)); + $this->assertEquals($string, Database::_sanitizeClob($clob)); + } }