From 2fa1f64869bb3a64491db98991f42c4acd0d2751 Mon Sep 17 00:00:00 2001 From: paulr Date: Sun, 31 Jul 2011 02:18:36 +1000 Subject: [PATCH] various chagnes --- plugins/repo.php | 323 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 298 insertions(+), 25 deletions(-) diff --git a/plugins/repo.php b/plugins/repo.php index 8512e12..522eef7 100644 --- a/plugins/repo.php +++ b/plugins/repo.php @@ -63,7 +63,7 @@ class GLCASRepo { * * I dont want to code this from scratch, but i probably need to */ - function getRepoForUrlNew($url) + function getRepoForUrl($url) { $xurl = split("[/,]", $url); @@ -145,17 +145,17 @@ class GLCASRepo { // check if the file exists and serve it up if(file_exists($actualfile)) { - $this->serveUpFile($actualFile, $matched); + $this->serveUpFile($actualfile, $matched); return; } else { // the file does not exist, we now need to go into "download" mode $remoteurl = $uconf[$matched]["url"]."/$file"; - $this->downloadAndServe($actualFile, $matched, $remoteurl); + $this->downloadAndServe($actualfile, $matched, $remoteurl); return; } } - function serveUpFile($filename, $repoid) + function serveUpFile($actualfile, $repoid) { $uconf = unserialize($this->config->getConfigVar("repodata")); $repostore = $this->config->getConfigVar("storagelocation"); @@ -163,7 +163,7 @@ class GLCASRepo { // figure out the range header garbage that centos/redhat send if(isset($_SERVER["HTTP_RANGE"])) { // we're using ranges - screw you stupid installer - $pr_range = preg_split("/[:\-, ]+/", $_SERVER["HTTP_RANGE"]); + $pr_range = preg_split("/[:\-=, ]+/", $_SERVER["HTTP_RANGE"]); // cut up ranges $rangestart = $pr_range[1]; @@ -174,6 +174,8 @@ class GLCASRepo { // now spit some headers header("HTTP/1.1 206 Partial Content"); header("Content-Length: ".$rangelength); + + header("Content-Range: bytes $rangesstr/".filesize($actualfile)); // determine mime type @@ -207,6 +209,10 @@ class GLCASRepo { // open the local file $localfile = fopen($actualfile, "r"); + if(!$localfile) { + error_log("normal upload went barf"); + return; + } // iterate over its length, send 8k at a time while(!feof($localfile)) { @@ -225,9 +231,13 @@ class GLCASRepo { } // TODO: this is the function im working on + // the alternative to this function is that if a file is in the process of being + // downloaded, we simply serve from upstream... not a good idea tho unless we create + // a local proxy right here function downloadAndServe($filename, $repoid, $remoteurl) { // this is important so downloads dont die + clearstatcache(); ignore_user_abort(true); // get the configurations we need @@ -237,51 +247,314 @@ class GLCASRepo { // this is the tricky one for ranges. // check if a download exists + $otherdownloader = false; if(file_exists("$filename.tmp.data.deleteme")) { // a download exists, does it still work + error_log("DOWNLOADER: file exists for current download, hope it works, attempting lock"); $localtmpfh = fopen("$filename.tmp.data.deleteme", "r"); $lockres = flock($localtmpfh, LOCK_EX|LOCK_NB); if(!$lockres) { error_log("flock did fail, all is right with the world a download is in progress"); + $otherdownloader = true; } else { + error_log("lock succeeded, dieing in the arse"); unlink("$filename.tmp.data.deleteme"); unlink("$filename.tmp.data.deleteme.size"); } } // open the remote file - $rf = fopen($remoteurl, "r"); - - - // get the headers from the remote request and use them to hurt people $contentlen = 0; - foreach($http_response_header as $key => $val) { - if(preg_match("/HTTP.*30[1-9].*/", $val)) { - error_log("got a 30x, must be a directory"); - mkdir($filename); - header("Location: ".$_SERVER["REQUEST_URI"]."/"); + $contenttype = ""; + if(!$otherdownloader) { + $remotefile = fopen($remoteurl, "r"); + $localfile = fopen($filename.".tmp.data.deleteme", "w"); + $lockres = flock($localfile, LOCK_EX); + if(!$localfile) { + erorr_log("something went plop"); return; } - // get content length form upstream and print - if(preg_match("/^Content-Length:.*/", $val)) { - // WARNING, THIS IS NOT RIGHT - $contentlen = $val; - header($val); + // get the headers from the remote request and use them to hurt people + $contentlen = 0; + foreach($http_response_header as $key => $val) { + if(preg_match("/HTTP.*30[1-9].*/", $val)) { + error_log("got a 30x, must be a directory"); + mkdir($filename); + header("Location: ".$_SERVER["REQUEST_URI"]."/"); + return; + } + // get content length form upstream and print + if(preg_match("/^Content-Length:.*/", $val)) { + // WARNING, THIS IS NOT RIGHT + $clentemp = preg_split("/[: ]+/", $val); + $contentlen = $clentemp[1]; + //header($val); + } + // get content type from upstream and print + if(preg_match("/^Content-Type:.*/", $val)) { + $contenttype = $val; + } + if(!$remotefile) { + header("HTTP/1.0 404 Not Found"); + error_log("asked fore file that dont exist"); + return; + } + } - // get content type from upstream and print - if(preg_match("/^Content-Type:.*/", $val)) { - header($val); + error_log("put contentlen as $contentlen"); + file_put_contents($filename.".tmp.data.deleteme.size", $contentlen); + } else { + $localfile = fopen($filename.".tmp.data.deleteme", "r"); + if(!$localfile) { + error_log("something went plop"); + return; } } + + + // determine if we're ranged + $ranged = false; + $rangestart = 0; + $rangelength = 0; + $rangestr=""; + if(isset($_SERVER["HTTP_RANGE"])) { + // we're using ranges - screw you stupid installer + + $pr_range = preg_split("/[:\-=, ]+/", $_SERVER["HTTP_RANGE"]); + error_log("got range ".$_SERVER["HTTP_RANGE"]." and ".print_r($pr_range, true)); + + // cut up ranges + $rangestart = $pr_range[1]; + $rangelength = $pr_range[2] - $pr_range[1] +1; + $rangestr = $pr_range[1]."-".$pr_range[2]; + error_log("going ranges at $rangestart, $rangelength,"); + $ranged = true; + } + // open the local files - $localfile = fopen($filename.".tmp.data.deleteme", "w"); - $localsizefile = fopen($filename.".tmp.data.deleteme.size", "w"); + // now, lets determine what state we're in + // we're either - getting and sending + // watching and sending + // or a range (Getting and sending) + // or a range (watching and sending) + // TODO: it may be advicable to start the download as a seperate cli process rather then something goin on here + // so it definitely cant be interrupted. + + + // first, getting and sending - this is easy. + if(!$ranged && !$otherdownloader) { + while(!feof($remotefile)) { + $data = fread($remotefile, 2048); + echo $data; + flush(); + fwrite($localfile, $data); + } + rename($filename.".tmp.data.deleteme", $filename); + unlink($filename.".tmp.data.deleteme.size"); + + // and we're done + return; + + + + + // IT WORKS!!!!!!! + } else if ($otherdownloader && !$ranged) { + // this is where the fun starts - but this one isnt too bad. + error_log("OTHERDOWNLOAD: im another downloader, please work"); + $fsize = file_get_contents($filename.".tmp.data.deleteme.size"); + header("Content-Length: $fsize"); + $sgotten = 0; + while(!feof($localfile)) { + error_log("OTHERDOWNLOAD: get"); + $data = fread($localfile, 2048); + if(!$data) { + error_log("dollardata is pair shaped"); + } else { + $sgotten += strlen($data); + if($sgotten > $fsize) { + error_log("went plop at sgotten, $sgotten, $fsize"); + return; + } + echo $data; + flush(); + } + } + fclose($localfile); + + // need to think about this in pseudo code. + // 1. close the file and wait for it to get to $sgotten + 2048 or $fsize + if(file_exists($filename.".tmp.data.deleteme")) $cursize = filesize($filename.".tmp.data.deleteme"); + else if(file_exists($filename)) { + error_log("DOTHERDOWNLOADER: namechange"); + $cursize = filesize($filename); + } else return; // we had to bail + + + $upload_finished = false; + while(!$upload_finished) { + while($cursize < $fsize && $cursize < ($sgotten+2048)) { + clearstatcache(); + error_log("OTHERDOWNLOAD: halt, $cursize, $sgotten, $fsize"); + // sleep until the the filesize is greater then what we're up to, or until the file is finished + sleep(1); + if(file_exists($filename.".tmp.data.deleteme")) { + error_log("OTHERDOWNLOADER: still same name"); + $cursize = filesize($filename.".tmp.data.deleteme"); + } else if(file_exists($filename)) { + error_log("DOTHERDOWNLOADER: namechange"); + $cursize = filesize($filename); + } + else return; // we had to bail + } + + error_log("OTHERDOWNLOAD: continue, $sgotten, $fsize"); + // reopen local file - if it stopped existing, we need to deal with that + if(file_exists($filename.".tmp.data.deleteme")) $localfile = fopen($filename.".tmp.data.deleteme", "r"); + else if(file_exists($filename)) $localfile = fopen($filename, "r"); + else return; // we had to bail + + // UG, we need to ff, how could i forget that + fseek($localfile, $sgotten); + + if(!$localfile) { + error_log("OTHERDOWNLOAD: something went plop"); + return; + } + + // now loop on the file until we have it at an eof + while(!feof($localfile)) { + $data = fread($localfile, 512); + if(!$data) { + error_log("OTHERDOWNLOAD: dollar data went plop"); + } else { + $sgotten += strlen($data); + echo $data; + flush(); + } + } + fclose($localfile); + + if($sgotten >= $fsize) { + if($sgotten > $fsize) error_log("OTHERDOWNLOADER: finished but $sgotten, $fsize doesnt make senze"); + $upload_finished = true; + } + // and we're done + + } + error_log("OTHERDOWNLOADER: done with"); + + return; + + + + + // THIS WAS JUST AWESOME CAUSE IT WORKS + // Next painful bit + } else if ($ranged && !$otherdownloader) { + + $sgotten = 0; + + + // the problem is here + error_log("Downloader: going ranged as primary"); + clearstatcache(); + header("HTTP/1.1 206 Partial Content"); + header("Content-Length: ".$rangelength); + header("Content-Range: bytes $rangestr/".$contentlen); + header("Content-Type: $contenttype"); + header("Connection: close"); + + // first, get up to $rangestart + while(!feof($remotefile) && ftell($remotefile) < $rangestart) { + if(($rangestart - ftell($remotefile)) < 2048) $rsize = $rangestart; + else $rsize = 2048; + $data = fread($remotefile, $rsize); + if(!$data) { + error_log("dollar data went plop"); + } else { + $sgotten += strlen($data); + flush(); + fwrite($localfile, $data); + } + } + + error_log("should now be at rangestart: ".ftell($remotefile)); + + // now start pumping out data until $rangelength + $sgatlen = $rangelength + $rangestart; + while(!feof($remotefile) && ftell($remotefile) < $sgatlen ) { + + // read only 2048 + $rsize = $sgatlen - ftell($remotefile); + if($rsize > 2048) $rsize = 2048; + + $data = fread($remotefile, $rsize); + if(!$data) { + error_log("dollar data went plop"); + } else { + echo $data; + $sgotten += strlen($data); + flush(); + fwrite($localfile, $data); + } + } + + // hopefully this works, the redhat/centos installer really is a terrible + // piece of code.. The reasons for this is that redhat in their minds decided + // to use ranges, and even though you send the range to the client, the client + // just keeps listening. and you can ignore the range request, it ignores you. + + error_log("should now be at rangeend: $sgatlen, $rangestart, ".ftell($remotefile)); + flush(); + + // now continue on as per normal - totally gunna work... + while(!feof($remotefile)) { + //error_log("back download"); + $data = fread($remotefile, 2048); + flush(); + fwrite($localfile, $data); + } + + + rename($filename.".tmp.data.deleteme", $filename); + unlink($filename.".tmp.data.deleteme.size"); + + // and we're done + return; + + + + + + // and here + } else if ($ranged && $otherdownloader) { + // and here too, yay, someone else is doing the + // download, but we're the retards getting a range + $sgotten = 0; + + + // the problem is here + error_log("Downloader: going ranged as primary"); + clearstatcache(); + header("HTTP/1.1 206 Partial Content"); + header("Content-Length: ".$rangelength); + header("Content-Range: bytes $rangestr/".$contentlen); + header("Content-Type: $contenttype"); + clearstatcache(); + + error_log("OTHERDOWNLOAD: im another downloader, please work for ranged"); + + } + + + return; } // this is a nightmare - function getRepoForUrl($url) + function getRepoForUrlOld($url) { // the way we breakdown a url is to explode it $xurl = split("[/,]", $url); -- 1.7.0.4