利用PHP的curl批量导出文件

Posted 2020-10-08 满血复活的回来写代码

场景

1.需要从网站中导出大量的文件，包括有图片格式，文档格式等附件内容。

2.分文件夹来导出不同的数据

<?php
header("Content-Type:text/html;charset=‘utf-8‘");
include ‘csv.php‘ ;
$fileData = array(
    ‘csv/310-new.csv‘ => 310,
    ‘csv/308-new.csv‘ => 308,
    ‘csv/297-new.csv‘ => 297,
     ‘csv/296-new.csv‘ => 296,
     ‘csv/295-new.csv‘ => 295,
     ‘csv/293-new.csv‘ => 293,
);

foreach ($fileData as $skey => $svalue) {
    echo "export {$skey}\n";
    if(!file_exists("data/".$svalue)){
        mkdir("data/".$svalue);
    }
    if(!file_exists($skey)){
        echo " file $skey is not exists \n";
        continue;
    }
    $csvreader = new CsvReader( $skey );
    $data = $csvreader->get_data();
    foreach ($data as $key => $value) {
        $url = (trim($value[0]));
        $name = trim($value[1]);
        if( empty($url) or empty($name) ){
            echo "url is empty \n";
            continue;
        }
        $name = str_replace("/" , "-" ,$name );//字符串替换，文件名中不能有特殊字符
        $ext =  substr($url, strrpos($url, ‘.‘)+1);//获取原始文件名的尾缀
        downImage($url ,$svalue, $name .  "." .  $ext  );
        echo "export $url \n";
    }
}

 function downImage($url , $sfolder , $filename ){
    //$filename = iconv("utf-8","GBK//IGNORE", $filename);
    $folder = __DIR__ . "/data/" ;
    $filename = $folder . "{$sfolder}/" . $filename  ;
    $ret =  https_request($url);
    saveFile($filename,$ret); //下载后好的文件名
}

 function saveFile($filename, $filecontent){
    $local_file = fopen($filename, ‘w‘);
    if (false !== $local_file){
        if (false !== fwrite($local_file, $filecontent)) {
            fclose($local_file);
        }
    }
}

function https_request($url){
    $curl = curl_init();
    curl_setopt($curl, CURLOPT_URL, $url);
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, FALSE);
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
    $output = curl_exec($curl);
    curl_close($curl);
    return $output;
}

csv.php  封装的类库为

<?php
class CsvReader {
    private $csv_file;
    private $spl_object = null;
    private $error;
    
    public function __construct($csv_file = ‘‘) {
        if($csv_file && file_exists($csv_file)) {
            $this->csv_file = $csv_file;
        }
    }
    
    public function set_csv_file($csv_file) {
        if(!$csv_file || !file_exists($csv_file)) {
            $this->error = ‘File invalid‘;
            return false;
        }
        $this->csv_file = $csv_file;
        $this->spl_object = null;
    }
    
    public function get_csv_file() {
        return $this->csv_file;
    }
    
    private function _file_valid($file = ‘‘) {
        $file = $file ? $file : $this->csv_file;
        if(!$file || !file_exists($file)) {
            return false;
        }
        if(!is_readable($file)) {
            return false;
        }
        return true;
    }
    
    private function _open_file() {
        if(!$this->_file_valid()) {
            $this->error = ‘File invalid‘;
            return false;
        }
        if($this->spl_object == null) {
            $this->spl_object = new SplFileObject($this->csv_file, ‘rb‘);
        }
        return true;
    }

    public function get_data($length = 0, $start = 0) {
        if(!$this->_open_file()) {
            return false;
        }
        $length = $length ? $length : $this->get_lines();
        $start = $start - 1;
        $start = ($start < 0) ? 0 : $start;
        $data = array();
        $this->spl_object->seek($start);
        while ($length-- && !$this->spl_object->eof()) {
            $data[] = $this->spl_object->fgetcsv();
            $this->spl_object->next();
        }
        return $data;
    }
    
    public function get_lines() {
        if(!$this->_open_file()) {
            return false;
        }
        $this->spl_object->seek(filesize($this->csv_file));
        return $this->spl_object->key();
    }
    
    public function get_error() {
        return $this->error;
    }
}