X-Git-Url: http://git.pjr.cc/?a=blobdiff_plain;f=plugins%2Frepo.php;h=1546e61a10f19e2c84d6746e1269bcf5233cc4e9;hb=37b5d51a06265cba25c1c92418223837c6d3ad2b;hp=8512e12b567cec9d8a568d5a71894a7593a7ad0e;hpb=733ff7f20d87e104ae0b528602c946ec63fe137b;p=glcas.git diff --git a/plugins/repo.php b/plugins/repo.php index 8512e12..1546e61 100644 --- a/plugins/repo.php +++ b/plugins/repo.php @@ -1,26 +1,112 @@ config = $config; - if($this->config->getConfigVar("storagelocation") == false) { + if($this->config->getConfig("storagelocation") == false) { global $WEB_ROOT_FS; $storloc = "$WEB_ROOT_FS/../var/glcas/cache/"; if(!file_exists($storloc)) mkdir($storloc); - $this->config->setConfigVar("storagelocation", realpath($storloc)); - $this->config->saveConfig(); + $this->config->setConfig("storagelocation", realpath($storloc)); error_log("set storage location, $storloc"); } } + + // with the move to the sqlite db, this is going to get painful + function cron() + { + //echo "
";
+		$uconf = unserialize($this->config->getConfig("repodata"));
+		$repostore = $this->config->getConfig("storagelocation");
+		//print_r($uconf);
+		//echo "
"; + + foreach($uconf as $rkey => $repo) { + echo "Repo $rkey: ".$repo["desc"]."
"; + if(isset($repo["frozen"])) { + error_log("repo froze set"); + $repofrozen = $repo["frozen"]; + } else { + error_log("repo froze not set"); + $repofrozen = false; + } + if(!isset($repo["expiretime"])) { + echo " - Expire time not set, setting to 2 days by default
"; + $uconf[$rkey]["expiretime"] = 2; + $repo["expiretime"] = 2; + $this->config->setConfig("repodata", serialize($uconf)); + $this->config->saveConfig(); + } + + if(!isset($repo["repotype"])) { + echo "Repo type not set for repo, setting to yum
"; + $uconf[$rkey]["repotype"] = "YUM"; + $this->config->setConfig("repodata", serialize($uconf)); + $this->config->saveConfig(); + } else if($repo["repotype"] == "YUM") { + if($repofrozen) { + error_log("repo frozen"); + echo " - Repo Frozen
"; + } else { + if(file_exists("$repostore/$rkey/repodata/repoupdate.lock")) { + echo " - Repo locked for update
"; + + } + + // we still do this next bit, even if its locked because it may be a stray file + echo " - Expire time is ".$repo["expiretime"]." days, checking repo
"; + error_log("checking $repostore/$rkey/repodata/repomd.xml"); + $tdiff = time() - filemtime("$repostore/$rkey/repodata/repomd.xml"); + $maxtdiff = $repo["expiretime"] * 24 * 3600; + if($tdiff > $maxtdiff) { + echo " - updating repo
"; + $this->updateRepo($rkey); + } else { + echo " - not updating repo
"; + } + } + } else if($repo["repotype"] == "APT") { + if($repofrozen) { + error_log("repo frozen"); + echo " - Repo Frozen
"; + } else { + if(file_exists("$repostore/$rkey/update.lock")) { + echo " - Repo locaked for update
"; + } + echo " - Expire time is ".$repo["expiretime"]." days, checking repo
"; + $maxtdiff = $repo["expiretime"] * 24 * 3600; + // now we check for outdated release files + $dh = opendir("$repostore/$rkey/dists/"); + $needsupdate = false; + while(($fn = readdir($dh))!==false) { + if($fn != "." && $fn != "..") { + if(file_exists("$repostore/$rkey/dists/$fn/Release")) { + $tdiff = time() - filemtime("$repostore/$rkey/dists/$fn/Release"); + if($tdiff > $maxtdiff) $needsupdate = true; + } + } + } + if($needsupdate) { + echo " - Spawn updater
"; + $this->updateRepo($rkey); + } else { + echo " - Up to date
"; + } + } + } + } + } + function go($url) { error_log("repo:go called"); - - // figure out what we're doing + + // figure out what we're doing switch($url) { case "list": GLCASpageBuilder($this, "body"); @@ -29,7 +115,7 @@ class GLCASRepo { $this->getRepoForUrl($url); } } - + function body($url) { // this is how this will work @@ -40,40 +126,40 @@ class GLCASRepo { } echo "i am the repo, $url"; } - - + + // TODO: rework this function /* - * What i need to do is have a downloader function - * that can cope with lots of different shit - * but thats a pipe dream - * - * what *THIS* function needs to do is - * 1) figure out the repo - * 2) figure out the file in the repo - * 2.1) if its a directory, go to print directory - * 3) if the file exists, give it to the user (if a range is specified give the user the range) - * 4) if the file does not exist - * - check if a tmp file exists - * - attempt to get an exclusive flock - * - if flock fails, donwload in progress - * - if flock succeeds, truncate file and re-start download - * - if a range request was made, send the range once available - * - if range not available, sleep for 5 and check again. - * - * I dont want to code this from scratch, but i probably need to - */ - function getRepoForUrlNew($url) + * What i need to do is have a downloader function + * that can cope with lots of different shit + * but thats a pipe dream + * + * what *THIS* function needs to do is + * 1) figure out the repo + * 2) figure out the file in the repo + * 2.1) if its a directory, go to print directory + * 3) if the file exists, give it to the user (if a range is specified give the user the range) + * 4) if the file does not exist + * - check if a tmp file exists + * - attempt to get an exclusive flock + * - if flock fails, donwload in progress + * - if flock succeeds, truncate file and re-start download + * - if a range request was made, send the range once available + * - if range not available, sleep for 5 and check again. + * + * I dont want to code this from scratch, but i probably need to + */ + function getRepoForUrl($url) { $xurl = split("[/,]", $url); - - // first get the config - $uconf = unserialize($this->config->getConfigVar("repodata")); - $repostore = $this->config->getConfigVar("storagelocation"); - + + // first get the config + $uconf = unserialize($this->config->getConfig("repodata")); + $repostore = $this->config->getConfig("storagelocation"); + // preset matched to -1 $matched = -1; - + // first we check for /repo/repoid as a url $startat = 0; if($xurl[0] == "repo") { @@ -85,12 +171,12 @@ class GLCASRepo { $startat +=2; } } - + // now check for a prefix match $prematch = false; if($matched < 0) foreach($uconf as $key => $var) { $pre = $var["prefix"]; - + if($pre!="") { //echo "Checking pre $pre against ".$xurl[0]."\n"; if(strcasecmp($pre, $xurl[0])==0) { @@ -100,13 +186,13 @@ class GLCASRepo { } } } - + // next, check for a short url match if($matched < 0) foreach($uconf as $key => $var) { // if we matched a pre, then we check against the second url component - + $short = $var["shorturl"]; - + if($short!="") { //echo "Checking short $short against ".$xurl[$startat]."\n"; if(strcasecmp($xurl[$startat], $short)==0) { @@ -116,7 +202,7 @@ class GLCASRepo { } } } - + // TODO: this deterministic bit // so far nothing has matched - what this next bit needs to do is try and "Determine" a repo from url // for eg, if a user gets /fedora/x86_64/os we need to return something appropriate @@ -126,73 +212,89 @@ class GLCASRepo { return; } - + + // something was matched, so now we reconstruct the file component of the url $file = "/"; if(count($xurl) > $startat) for($i=$startat; $i < count($xurl); $i++) { $file .= "/".$xurl[$i]; } - + // so, the ultimate url for the file we need is: $actualfile = "$repostore/$matched/$file"; error_log("Atcualfile is $actualfile"); + // now check for a block in that repo + if(isset($uconf[$matched]["blocklist"])) { + foreach($uconf[$matched]["blocklist"] as $blockers) { + $rfile = ltrim($file, "/"); + error_log("checking $blockers against $rfile"); + if(preg_match("/$blockers/", $rfile) > 0) { + error_log("should block"); + header("HTTP/1.0 404 Not Found"); + return; + } + } + } + // if its a directory, lets do a print if(is_dir($actualfile)) { $this->printDir($actualfile, $file, $url); return; } - + // check if the file exists and serve it up - if(file_exists($actualfile)) { - $this->serveUpFile($actualFile, $matched); + if(file_exists($actualfile) && !file_exists("$actualfile.size")) { + $this->serveUpFile($actualfile, $matched); return; } else { // the file does not exist, we now need to go into "download" mode $remoteurl = $uconf[$matched]["url"]."/$file"; - $this->downloadAndServe($actualFile, $matched, $remoteurl); + $this->downloadAndServe($actualfile, $matched, $remoteurl); return; } } - - function serveUpFile($filename, $repoid) + + function serveUpFile($actualfile, $repoid) { - $uconf = unserialize($this->config->getConfigVar("repodata")); - $repostore = $this->config->getConfigVar("storagelocation"); - + $uconf = unserialize($this->config->getConfig("repodata")); + $repostore = $this->config->getConfig("storagelocation"); + // figure out the range header garbage that centos/redhat send if(isset($_SERVER["HTTP_RANGE"])) { // we're using ranges - screw you stupid installer - $pr_range = preg_split("/[:\-, ]+/", $_SERVER["HTTP_RANGE"]); - + $pr_range = preg_split("/[:\-=, ]+/", $_SERVER["HTTP_RANGE"]); + // cut up ranges $rangestart = $pr_range[1]; $rangelength = $pr_range[2] - $pr_range[1] +1; $rangestr = $pr_range[1]."-".$pr_range[2]; error_log("going ranges at $rangestart, $rangelength,".$rangesa[1].",".$rangesb[0]); - + // now spit some headers header("HTTP/1.1 206 Partial Content"); header("Content-Length: ".$rangelength); - header("Content-Range: bytes $rangesstr/".filesize($actualfile)); - + + + header("Content-Range: bytes $rangestr/".filesize($actualfile)); + // determine mime type $type = mime_content_type($actualfile); - + // set mime type header header("Content-type: $type"); - + // open the local file (TODO: error check) $localfile = fopen($actualfile, "r"); fseek($localfile, $rangestart, SEEK_SET); - + // read in the data, god i hope its not big $data = fread($localfile, $rangelength); - + // lastly, send data echo $data; flush(); - + // and close the file fclose($localfile); return; @@ -204,284 +306,299 @@ class GLCASRepo { // set the mime type header $type = mime_content_type($actualfile); header("Content-type: $type"); - - // open the local file + + // open the local file $localfile = fopen($actualfile, "r"); - + if(!$localfile) { + error_log("normal upload went barf"); + return; + } + // iterate over its length, send 8k at a time while(!feof($localfile)) { // read and send data - $data = fread($localfile, 8192); + $data = fread($localfile, 32768); echo $data; - + // flush so the client sees the data flush(); } - + // close the file fclose($localfile); return; - } + } } - + // TODO: this is the function im working on + // the alternative to this function is that if a file is in the process of being + // downloaded, we simply serve from upstream... not a good idea tho unless we create + // a local proxy right here - this function is a race condition waiting to be had + // lets hope its a good one! function downloadAndServe($filename, $repoid, $remoteurl) { - // this is important so downloads dont die - ignore_user_abort(true); - + + $this->startDownload($filename, $remoteurl); + + // give the proc a minute to get going + sleep(2); + clearstatcache(); + // get the configurations we need - $uconf = unserialize($this->config->getConfigVar("repodata")); - $repostore = $this->config->getConfigVar("storagelocation"); - - // this is the tricky one for ranges. - - // check if a download exists - if(file_exists("$filename.tmp.data.deleteme")) { - // a download exists, does it still work - $localtmpfh = fopen("$filename.tmp.data.deleteme", "r"); - $lockres = flock($localtmpfh, LOCK_EX|LOCK_NB); - if(!$lockres) { - error_log("flock did fail, all is right with the world a download is in progress"); - } else { - unlink("$filename.tmp.data.deleteme"); - unlink("$filename.tmp.data.deleteme.size"); - } - } + $uconf = unserialize($this->config->getConfig("repodata")); + $repostore = $this->config->getConfig("storagelocation"); - // open the remote file - $rf = fopen($remoteurl, "r"); - - - // get the headers from the remote request and use them to hurt people - $contentlen = 0; - foreach($http_response_header as $key => $val) { - if(preg_match("/HTTP.*30[1-9].*/", $val)) { - error_log("got a 30x, must be a directory"); - mkdir($filename); - header("Location: ".$_SERVER["REQUEST_URI"]."/"); - return; - } - // get content length form upstream and print - if(preg_match("/^Content-Length:.*/", $val)) { - // WARNING, THIS IS NOT RIGHT - $contentlen = $val; - header($val); - } - // get content type from upstream and print - if(preg_match("/^Content-Type:.*/", $val)) { - header($val); - } + + + // determine if we're ranged + $ranged = false; + $rangestart = 0; + $rangelength = 0; + $rangestr=""; + if(isset($_SERVER["HTTP_RANGE"])) { + // we're using ranges - screw you stupid installer + + $pr_range = preg_split("/[:\-=, ]+/", $_SERVER["HTTP_RANGE"]); + error_log("got range ".$_SERVER["HTTP_RANGE"]." and ".print_r($pr_range, true)); + + // cut up ranges + $rangestart = $pr_range[1]; + $rangelength = $pr_range[2] - $pr_range[1] +1; + $rangestr = $pr_range[1]."-".$pr_range[2]; + error_log("going ranges at $rangestart, $rangelength, $rangestr"); + $ranged = true; } - + // open the local files - $localfile = fopen($filename.".tmp.data.deleteme", "w"); - $localsizefile = fopen($filename.".tmp.data.deleteme.size", "w"); - - } - - // this is a nightmare - function getRepoForUrl($url) - { - // the way we breakdown a url is to explode it - $xurl = split("[/,]", $url); - - // we first check if [0] is a prefix - // if now, we check for it being a shorturl (lets just do that for now) - $uconf = unserialize($this->config->getConfigVar("repodata")); - $repostore = $this->config->getConfigVar("storagelocation"); - - $matched = -1; - - // first we check for /repo/repoid as a url - $startat = 0; - if($xurl[0] == "repo") { - $repid = $xurl[1]; - error_log("trying to get repo for repoid, $repid"); - if(isset($uconf[$repid])) { - $matched = ((int)($repid)); - error_log("set matched, $matched, $repid"); - $startat +=2; - } - } - - - $prematch = false; - if($matched < 0) foreach($uconf as $key => $var) { - $pre = $var["prefix"]; + + // now, lets determine what state we're in + // we're either - getting and sending + // watching and sending + // or a range (Getting and sending) + // or a range (watching and sending) + // TODO: it may be advicable to start the download as a seperate cli process rather then something goin on here + // so it definitely cant be interrupted. + + // check for a 404 file and wait 2 if it exists - i should really check the timestamp for an updated + // file, but thats too much effort for now: TODO: check timestamp on 404 file + $slept = 0; + while(!file_exists("$filename")) { + clearstatcache(); + sleep(1); + $slept += 1; + error_log("Sleeping waiting for file"); - if($pre!="") { - //echo "Checking pre $pre against ".$xurl[0]."\n"; - if(strcasecmp($pre, $xurl[0])==0) { - //echo "Matched pre\n"; - $prematch = true; - $startat++; - } + // if 404 file exists, we wait much less time + if(file_exists("$filename.404") && $slept > 2) { + header("HTTP/1.0 404 Not Found"); + return; } + if($slept > 10) { + header("HTTP/1.0 404 Not Found"); + return; + } } - - if($matched < 0) foreach($uconf as $key => $var) { - // if we matched a pre, then we check against the second url component + clearstatcache(); + if(is_dir($filename)) { - $short = $var["shorturl"]; - - if($short!="") { - //echo "Checking short $short against ".$xurl[$startat]."\n"; - if(strcasecmp($xurl[$startat], $short)==0) { - //echo "Matched\n"; - $matched = $key; - $startat++; - } - } - } - - if($matched < 0) { - echo "No such repo
"; + header("Location: ".$_SERVER["REQUEST_URI"]."/"); return; } - - - // now we find an actual file - $file = "/"; - if(count($xurl) > $startat) for($i=$startat; $i < count($xurl); $i++) { - $file .= "/".$xurl[$i]; - } - - // now we want to find repostore/$matched/$file; - $actualfile = "$repostore/$matched/$file"; - error_log("Atcualfile is $actualfile"); - //echo "Start file for $actualfile\n"; - - // first check any directories in $file are in existence - $splfile = explode("/", $file); - if(count($splfile) > 1) { - $tomake = "$repostore/$matched/"; - for($i = 0; $i < count($splfile)-1; $i++) { - $tomake .= "/".$splfile[$i]; - //error_log("making directory $tomake"); - if(!is_dir($tomake)) mkdir($tomake); - } - } - - $reqhead = print_r($_REQUEST, true); - $sevhead = print_r($_SERVER, true); - - error_log("req $reqhead"); - error_log("sev $sevhead"); - - $rangestart = -1; - $rangelength = -1; - $rangesstr = ""; - if(isset($_SERVER["HTTP_RANGE"])) { - // oh shit - $rangesa = explode("=", $_SERVER["HTTP_RANGE"]); - $rangesb = explode(",", $rangesa[1]); - $rangesstr = $rangesb[0]; - $ranges = explode("-", $rangesb[0]); - $rangestart = $ranges[0]; - $rangelength = $ranges[1] - $ranges[0] +1; - error_log("going ranges at $rangestart, $rangelength,".$rangesa[1].",".$rangesb[0]); - } - - // i have to support http_range cause REDHAT/CENTOS IS annoying as all hell. christ, why do this? - if(is_file($actualfile)) { - // file is stored locally, away we go - if($rangelength != -1) { - header("HTTP/1.1 206 Partial Content"); - header("Content-Length: ".$rangelength); - header("Content-Range: bytes $rangesstr/".filesize($actualfile)); - //header("Content-Length: ".filesize($actualfile)); - } else { - header("Content-Length: ".filesize($actualfile)); - } - $type = mime_content_type($actualfile); - header("Content-type: $type"); - $localfile = fopen($actualfile, "r"); - if($rangestart!=-1) fseek($localfile, $rangestart, SEEK_SET); + + + // first, getting and sending - this is easy. + if (!$ranged) { + + $localfile = fopen($filename, "r"); + + // this is where the fun starts - but this one isnt too bad. + error_log("OTHERDOWNLOAD: im another downloader, please work"); + if(file_exists("$filename.size")) $fsize = file_get_contents("$filename.size"); + else $fsize = filesize($filename); + header("Content-Length: $fsize"); + $sgotten = 0; while(!feof($localfile)) { - // cant make this high cause centos is crap - if($rangelength!=-1) { - $data = fread($localfile, $rangelength); - error_log("data size was ".strlen($data)); + $data = fread($localfile, 2048); + if(!$data) { + error_log("dollardata is pair shaped"); } else { - $data = fread($localfile, 2048); + $sgotten += strlen($data); + if($sgotten > $fsize) { + error_log("went plop at sgotten, $sgotten, $fsize"); + return; + } + echo $data; + flush(); } + } + fclose($localfile); - echo $data; - flush(); + // need to think about this in pseudo code. + // 1. close the file and wait for it to get to $sgotten + 2048 or $fsize + $cursize = filesize($filename); - if($rangelength!=-1) { - fclose($localfile); - exit(0); + $upload_finished = false; + while(!$upload_finished) { + while($cursize < $fsize && $cursize < ($sgotten+2048)) { + clearstatcache(); + error_log("OTHERDOWNLOAD: halt, $cursize, $sgotten, $fsize"); + // sleep until the the filesize is greater then what we're up to, or until the file is finished + sleep(1); + $cursize = filesize($filename); } - } - fclose($localfile); - } else if(is_dir($actualfile)) { - //echo "in dir for $actualfile\n"; - // here we print the contents of the directory - $this->printDir($actualfile, $file, $url); - } else { - // ok, get the file - //echo "in getcheck\n"; - $remotefile = $uconf[$matched]["url"]."/$file"; - - // TODO: i should get remote contents with fopen/fread/fwrite as - // it should be more memory conservative and we can push to the end client - // straight away - ignore_user_abort(true); - $rf = fopen($remotefile, "r"); - error_log("attempting to get remote file $remotefile"); - - // hopefully this works. if we get a 30x message, it means we tried to get a directory - // i cant think of another way of dealing with it - but this is UGLY - // also get content length and content type - $clen = 0; - foreach($http_response_header as $key => $val) { - if(preg_match("/HTTP.*30[1-9].*/", $val)) { - error_log("got a 30x, must be a directory"); - mkdir($actualfile); - header("Location: ".$_SERVER["REQUEST_URI"]."/"); + error_log("OTHERDOWNLOAD: continue, $sgotten, $fsize"); + // reopen local file - if it stopped existing, we need to deal with that + $localfile = fopen($filename, "r"); + + // UG, we need to ff, how could i forget that + fseek($localfile, $sgotten); + + if(!$localfile) { + error_log("OTHERDOWNLOAD: something went plop"); return; } - // get content length form upstream and print - if(preg_match("/^Content-Length:.*/", $val)) { - $clen = $val; - header($val); + + // now loop on the file until we have it at an eof + while(!feof($localfile)) { + $data = fread($localfile, 512); + if(!$data) { + error_log("OTHERDOWNLOAD: dollar data went plop"); + } else { + $sgotten += strlen($data); + echo $data; + flush(); + } } - // get content type from upstream and print - if(preg_match("/^Content-Type:.*/", $val)) { - header($val); + fclose($localfile); + + if($sgotten >= $fsize) { + if($sgotten > $fsize) error_log("OTHERDOWNLOADER: finished but $sgotten, $fsize doesnt make senze"); + $upload_finished = true; } + // and we're done + } - //error_log("repsonse: $http_response_header"); - if(!$rf) { - // return 404 - header("HTTP/1.0 404 Not Found"); - } else { - $localfile = fopen($actualfile.".tmp.data.deleteme", "w"); - $localsizefile = fopen($actualfile.".tmp.data.deleteme.size", "w"); - fwrite($localsizefile, "$clen"); - fclose($localsizefile); - while(!feof($rf)) { - $data = fread($rf, 8192); - echo $data; - fwrite($localfile, $data); - flush(); + error_log("OTHERDOWNLOADER: done with"); + + return; + + + + + // Next painful bit + } else { + // and here too, yay, someone else is doing the + // download, but we're the retards getting a range + $sgotten = 0; + + $sgatlen = $rangestart+$rangelength; + + // the problem is here + error_log("Downloader: going ranged as other"); + clearstatcache(); + if(file_exists($filename.".tmp.data.deleteme.size")) $contentlen = file_get_contents($filename.".tmp.data.deleteme.size"); + else $contentlen = filesize($filename); + $contenttype = mime_content_type($filename); + header("HTTP/1.1 206 Partial Content"); + header("Content-Length: $rangelength"); + header("Content-Range: bytes $rangestr/$contentlen"); + $contenttype = "Content-Type: application/x-rpm"; + + error_log("$contenttype"); + header("$contenttype"); + + + clearstatcache(); + + + // first we wait until the file reaches $rangestart + while(filesize("$filename") < $rangestart) { + sleep(1); + } + + // then we open the file and ff to rangestart + $localfile = fopen($filename, "r"); + fseek($localfile, $rangestart); + + $sgotten = 0; + // need to think about this in pseudo code. + // 1. close the file and wait for it to get to $sgotten + 2048 or $fsize + $cursize = filesize($filename); + + + $upload_finished = false; + while(!$upload_finished) { + while($cursize < $sgatlen && $cursize < ($sgotten+2048)) { + clearstatcache(); + error_log("OTHERDOWNLOAD: halt, $cursize, $sgotten, $contentlen"); + // sleep until the the filesize is greater then what we're up to, or until the file is finished + sleep(1); + $cursize = filesize($filename); + } + + error_log("OTHERDOWNLOAD: continue, $sgotten, $contentlen"); + // reopen local file - if it stopped existing, we need to deal with that + $localfile = fopen($filename, "r"); + + // UG, we need to ff, how could i forget that + fseek($localfile, $sgotten+$rangestart); + + if(!$localfile) { + error_log("OTHERDOWNLOAD: something went plop"); + return; + } + + // now loop on the file until we have it at sgatlen + while(!feof($localfile) && $sgotten < $rangelength) { + $left = $rangelength - $sgotten; + if($left > 512) $lenget = 512; + else $lenget = $left; + $data = fread($localfile, $lenget); + if(!$data) { + error_log("OTHERDOWNLOAD: dollar data went plop"); + } else { + $sgotten += strlen($data); + echo $data; + flush(); + } } fclose($localfile); - fclose($rf); - rename($actualfile.".tmp.data.deleteme", $actualfile); - //error_log("got actualfile, tried to save as $actualfile, did it work?"); + + if($sgotten >= $rangelength) { + if($sgotten > $rangelength) error_log("OTHERDOWNLOADER: finished but $sgotten, $fsize doesnt make senze"); + $upload_finished = true; + } + // and we're done + } + error_log("OTHERDOWNLOADER: done with"); + + return; + } + + + return; + } + + function startDownload($file, $url) + { - //echo "got ".$file." for $url which is $actualfile\n"; - - //echo ""; + error_log("background downloader, start"); + global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; + if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/downloadfile.php '$url' '$file' > /tmp/dllog 2>&1 &"; + system($scall); + } else { + error_log("cant find download helper... dieing"); + } } - + + // this is a nightmare + + function printDir($dir, $localfile, $baseurl) { $localfile = preg_replace("/\/\/+/", "/", $localfile); @@ -491,32 +608,97 @@ class GLCASRepo { $content .= "Index of $localfile

Index of $localfile

"; $content .= ""; $dh = opendir($dir); + $dirn = 0; + $filen = 0; while(($file = readdir($dh))!==false) { - if($file != "." && $file != "..") $content .= ""; + if($file != "." && $file != "..") { + if(is_dir("$dir/$file")) { + $dirlist[$dirn++] = "$file"; + } else { + $filelist[$filen++] = "$file"; + } + } + } + if(isset($dirlist)) { + sort($dirlist); + foreach($dirlist as $dirs) { + $icon = "/icons/folder.png"; + $content .= ""; + } + } + if(isset($filelist)) { + sort($filelist); + foreach($filelist as $files) { + $fsize = filesize("$dir/$files"); + $icon = "/icons/text.png"; + $content .= ""; + } } $content .= "
$file
$dirs
$files$fsize
"; - + GLCASpageBuilder(null, null, $content); - + } else return false; } + function getRepoDetailsApt($url) + { + $action1 = $url."/dists"; + + // we just want to make sure it exists really + error_log("in repo details apt for $url"); + if(!glcas_isRemoteDir($action1)) { + //echo "I cant find any valid APT dists's at $url
"; + return false; + } + + // ok, now scan for ubuntu dists as + $kos = getKnownOSList(); + + $repos = 0; + $existing_repo["isrepo"] = true; + foreach($kos["apt"] as $key => $val) { + //echo "
$key, $val
"; + //echo "now check, $action1/$key"; + if(glcas_isRemoteDir($action1."/$key")) { + $existing_repos["knownrepo"][$repos]["name"] = $key; + //echo "Found Distro $val
"; + if(glcas_fileExists($action1."/$key/Contents-amd64.gz")) $existing_repos["knownrepo"][$repos]["amd64"] = true; + else $existing_repos["knownrepo"][$repos]["amd64"] = false; + if(glcas_fileExists($action1."/$key/Contents-i386.gz")) $existing_repos["knownrepo"][$repos]["i386"] = true; + else $existing_repos["knownrepo"][$repos]["i386"] = false; + $repos++; + + } + } + $existing_repos["nrepos"] = $repos; + + // TODO: these need to be "calculated" + $existing_repos["distros"] = "Ubuntu, Debian"; + $existing_repos["versions"] = "8.04LTS, 9.10, 10.04LTS, 10.10, 11.04, 11.10"; + $existing_repos["arch"] = "x86_64, i386"; + + + return $existing_repos; + + } + function getRepoDetailsYum($url, $ismirrorlist=false) { $actionurl = $url."/repodata/repomd.xml"; - + error_log("Getting for action of $actionurl"); - + $ld = file_get_contents($actionurl); - + // so here we try and get what this repository provides (os, version, arch), for yum this // should come straight off the url... i.e. centos/6.0/os/x86_64/ (centos, 6.0, base os, 64bit arch) - + if(!$ld) return false; - + // ok, now we tokenize the url and try and guess at the content $spurl = explode("/", $url); - + // first, find the OS $kos = getKnownOSList(); $glt["OS"] = "unknown"; @@ -524,7 +706,7 @@ class GLCASRepo { $glt["arch"] = "unknown"; $glt["other"] = "unknown"; foreach($spurl as $comp) { - + // find a name foreach($kos["os"]["short"] as $kosname => $koslong) { //error_log("Comparing $kosname and $koslong with $comp"); @@ -534,13 +716,13 @@ class GLCASRepo { $glt["OS"] = $koslong; } } - + // find a version, we assume its going to be something [numbers] and a . (optional) if(preg_match("/^[0-9.]+$/", $comp)>0) { //error_log("version match of $comp"); $glt["version"] = $comp; } - + // now architecture, this can be either i?86 or x86_64 - can also be arm or otherwise, but lets just go with this for now foreach($kos["arch"] as $archinter => $archname ) { //error_log("Comparing $archinter, $archname with $comp"); @@ -549,23 +731,53 @@ class GLCASRepo { $glt["arch"] = $archname; } } - + // other is a bt harder, we really have to guess at this one if(strcasecmp("os", $comp) == 0) $glt["other"] = "OS"; if(strcasecmp("update", $comp) == 0) $glt["other"] = "Updates"; if(strcasecmp("updates", $comp) == 0) $glt["other"] = "Updates"; if(strcasecmp("everything", $comp) == 0) $glt["other"] = "OS"; } - + return $glt; } + + function freezeRepo($rkey) + { + + $uconf = $this->config->getConfig("repodata"); + $repostore = $this->config->getConfig("storagelocation"); + + if($uconf !== false) { + $conf = unserialize($uconf); + foreach($conf as $key => $vla) { + if($key == $rkey) { + if(isset($conf["$rkey"]["frozen"])) { + if($conf["$rkey"]["frozen"]) { + $conf["$rkey"]["frozen"] = false; + error_log("Unfreeze repo"); + } else { + $conf["$rkey"]["frozen"] = true; + error_log("freeze repo"); + } + } else { + $conf["$rkey"]["frozen"] = true; + } + $nconf = serialize($conf); + error_log("freeze repo as $rkey"); + $this->config->setConfig("repodata", $nconf); + $this->config->saveConfig(); + } + } + } + } function deleteRepo($rkey) { - $uconf = $this->config->getConfigVar("repodata"); - $repostore = $this->config->getConfigVar("storagelocation"); - + $uconf = $this->config->getConfig("repodata"); + $repostore = $this->config->getConfig("storagelocation"); + if($uconf !== false) { $conf = unserialize($uconf); foreach($conf as $key => $vla) { @@ -574,17 +786,17 @@ class GLCASRepo { $nconf = serialize($conf); system("rm -rf $repostore/$key"); error_log("remove repo as $rkey"); - $this->config->setConfigVar("repodata", $nconf); + $this->config->setConfig("repodata", $nconf); $this->config->saveConfig(); } } } } - - function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init) + + function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init, $expiretime, $blocklist=null) { - $uconf = $this->config->getConfigVar("repodata"); - + $uconf = $this->config->getConfig("repodata"); + $cs["desc"] = $desc; $cs["os"] = $os; $cs["version"] = $version; @@ -594,8 +806,12 @@ class GLCASRepo { $cs["prefix"] = $prefix; $cs["url"] = $repurl; $cs["repotype"] = $repotype; - - + $cs["expiretime"] = $expiretime; + if($blocklist != null) { + $cs["blocklist"] = $blocklist; + } + + $ckey = 0; if($uconf !== false) { $conf = unserialize($uconf); @@ -604,88 +820,115 @@ class GLCASRepo { } $ckey++; } - + $conf[$ckey] = $cs; - + $nconf = serialize($conf); - + error_log("add repo as $ckey"); - $this->config->setConfigVar("repodata", $nconf); + $this->config->setConfig("repodata", $nconf); $this->config->saveConfig(); - + // now create the base structure in the repo - $repostore = $this->config->getConfigVar("storagelocation"); - - + $repostore = $this->config->getConfig("storagelocation"); + + // now call update repo if($init) $this->updateRepoYum($ckey); } - + function updateRepo($repokey) { // we only do yum yet - $this->updateRepoYum($repokey); + $repod = $this->getRepo($repokey); + if(isset($repod["frozen"])) if($repod["frozen"]) { + error_log("in update repo for frozen repo, ignore update"); + return; + } + + error_log("in update repo"); + + if($repod["repotype"] == "YUM") $this->updateRepoYum($repokey); + if($repod["repotype"] == "APT") $this->updateRepoApt($repokey); } - function updateRepoYum($repokey) + function updateRepoApt($repokey) { - $repostore = $this->config->getConfigVar("storagelocation"); - + $repostore = $this->config->getConfig("storagelocation"); + $repod = $this->getRepo($repokey); - + $repourl = $repod["url"]; - + if(!file_exists("$repostore/$repokey")) { mkdir("$repostore/$repokey"); } + if(!file_exists("$repostore/$repokey/dists/")) { + mkdir("$repostore/$repokey/dists/"); + // no point trying to update if we have no dists directory + return; + } + + + error_log("background apt repo update, start"); + global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; + if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/updateaptrepo.php '$repourl' '$repostore/$repokey/' > /tmp/updateaptrepo.$repokey.log 2>&1 &"; + system($scall); + } else { + error_log("cant find download apt helper... dieing"); + } + } + + function updateRepoYum($repokey) + { + $repostore = $this->config->getConfig("storagelocation"); + + $repod = $this->getRepo($repokey); + + $repourl = $repod["url"]; + + if(!file_exists("$repostore/$repokey")) { + mkdir("$repostore/$repokey"); + } + if(!file_exists("$repostore/$repokey/repodata")) { mkdir("$repostore/$repokey/repodata"); } - //ignore_user_abort(true); - $actionurl = "$repourl/repodata/repomd.xml"; - $repomdxml = file_get_contents($actionurl); - file_put_contents("$repostore/$repokey/repodata/repomd.xml", $repomdxml); - - $xml = simplexml_load_file("$repostore/$repokey/repodata/repomd.xml"); - - - foreach($xml as $key => $var) { - //echo "for key $key has:\n"; - //print_r($var); - if($key == "data") { - $fileloc = $var->location["href"]; - if(!file_exists("$repostore/$repokey/$fileloc")) { - error_log("getting $fileloc for $repokey on $repourl"); - $dlfile = file_get_contents("$repourl/$fileloc"); - file_put_contents("$repostore/$repokey/$fileloc", $dlfile); - } else { - error_log("Not getting $fileloc because we already have it"); - } - } + error_log("background yum repo update, start"); + global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; + if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/updateyumrepo.php '$repourl' '$repostore/$repokey/' > /tmp/updateyumrepo.$repokey.log 2>&1 &"; + system($scall); + } else { + error_log("cant find download yum helper... dieing"); } + + + //ignore_user_abort(true); } - + function getRepo($id) { - $uconf = $this->config->getConfigVar("repodata"); + $uconf = $this->config->getConfig("repodata"); if($uconf !== false) { $lconf = unserialize($uconf); return $lconf[$id]; } else return false; - + } - + function getRepos() { - $uconf = $this->config->getConfigVar("repodata"); + $uconf = $this->config->getConfig("repodata"); if($uconf !== false) { return unserialize($uconf); } else return false; - + } - + private $config; }