Attachment #23901 for bug #47433

View | Details | Raw Unified | Return to bug 47433
Collapse All | Expand All





package org.apache.tools.ant.taskdefs;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.util.FileUtils;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;

import java.net.URL;
import java.net.URLConnection;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.util.FileUtils;

/**
 * Gets a particular file from a URL source.

    private static final int DOTS_PER_LINE = 50;
    private static final int BIG_BUFFER_SIZE = 100 * 1024;
    private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();
    private static final int REDIRECT_LIMIT = 25;
    
    private static final String HTTP = "http";
    private static final String HTTPS = "https";

    private URL source; // required
    private File dest; // required

    }

    private class GetThread extends Thread {

        private final boolean hasTimestamp;
        private final long timestamp;
        private final DownloadProgress progress;

        private BuildException exception = null;
        private InputStream is = null;
        private OutputStream os = null;
        private Set triedSources = new HashSet();
        private URLConnection connection;

        GetThread(boolean h, long t, DownloadProgress p, int l) {
            hasTimestamp = h;

        }

        private boolean get() throws IOException, BuildException {
            
            connection = openConnection(source);

            if (connection == null)
            {
                return false;
            }

            boolean downloadSucceeded = downloadFile();

            //if (and only if) the use file time option is set, then
            //the saved file now has its timestamp set to that of the
            //downloaded file
            if (downloadSucceeded && useTimestamp)  {
                updateTimeStamp();
            }
            
            return downloadSucceeded;
        }


        private boolean redirectionAllowed(URL aSource, URL aDest) {
            if (!(aSource.getProtocol().equals(aDest.getProtocol()) || (HTTP
                    .equals(aSource.getProtocol()) && HTTPS.equals(aDest
                    .getProtocol())))) {
                String message = "Redirection detected from "
                        + aSource.getProtocol() + " to " + aDest.getProtocol()
                        + ". Protocol switch unsafe, not allowed.";
                if (ignoreErrors) {
                    log(message, logLevel);
                    return false;
                } else {
                    throw new BuildException(message);
                }
            }
            if (triedSources.contains(aDest.toString())) {
                String message = "Cyclic redirection detected: " + aDest;
                if (ignoreErrors) {
                    log(message, logLevel);
                    return false;
                } else {
                    throw new BuildException(message);
                }
            }

            if (triedSources.size() > REDIRECT_LIMIT) {
                String message = "More than " + REDIRECT_LIMIT
                        + " times redirected, giving up";
                if (ignoreErrors) {
                    log(message, logLevel);
                    return false;
                } else {
                    throw new BuildException(message);
                }
            }

            triedSources.add(aDest);
            return true;
        }

        private URLConnection openConnection(URL aSource) throws IOException {

            // set up the URL connection
            URLConnection connection = aSource.openConnection();
            // modify the headers
            // NB: things like user authentication could go in here too.
            if (hasTimestamp) {
                connection.setIfModifiedSince(timestamp);
            }

            if (uname != null || pword != null) {
                String up = uname + ":" + pword;
                String encoding;
                // we do not use the sun impl for portability,
                // and always use our own implementation for consistent
                // testing
                Base64Converter encoder = new Base64Converter();
                encoding = encoder.encode(up.getBytes());
                connection.setRequestProperty("Authorization", "Basic "
                        + encoding);
            }

            if (connection instanceof HttpURLConnection) {
                ((HttpURLConnection) connection)
                        .setInstanceFollowRedirects(false);
            }
            // connect to the remote site (may take some time)
            connection.connect();

            // First check on a 301 / 302 (moved) response
            if (connection instanceof HttpURLConnection) {
                HttpURLConnection httpConnection = (HttpURLConnection) connection;
              //  httpConnection.setInstanceFollowRedirects(false);
              //  httpConnection.setUseCaches(false);
                int responseCode = httpConnection.getResponseCode();
                if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || 
                        responseCode == HttpURLConnection.HTTP_MOVED_TEMP)
                {
                    String newLocation = httpConnection.getHeaderField("Location");
                    String message = aSource
                            + (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
                                    : "") + " moved to " + newLocation;
                    log(message, logLevel);
                    URL newURL = new URL(newLocation);
                    if (!redirectionAllowed(aSource, newURL))
                    {
                        return null;
                    }
                    return openConnection(newURL);
                }
                // next test for a 304 result (HTTP only)
                long lastModified = httpConnection.getLastModified();
                if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
                        || (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
                    // not modified so no file download. just return
                    // instead and trace out something so the user
                    // doesn't think that the download happened when it
                    // didn't
                    //doesn't think that the download happened when it
                    //didn't
                    log("Not modified - so not downloaded", logLevel);
                    return null;
                }
                // test for 401 result (HTTP only)
                if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
                    == HttpURLConnection.HTTP_UNAUTHORIZED)  {
                    String message = "HTTP Authorization failure";
                    if (ignoreErrors) {
                        log(message, logLevel);
                        return null;
                    } else {
                        throw new BuildException(message);
                    }
                }

            }

            //REVISIT: at this point even non HTTP connections may

            //the date of the content and skip the write if it is not
            //newer. Some protocols (FTP) don't include dates, of
            //course.
            return connection;
        }

        private boolean downloadFile()
                throws FileNotFoundException, IOException {
            for (int i = 0; i < NUMBER_RETRIES; i++) {
                // this three attempt trick is to get round quirks in different
                // Java implementations. Some of them take a few goes to bind
                // property; we ignore the first couple of such failures.
                try {
                    is = connection.getInputStream();
                    break;

                if (ignoreErrors) {
                    return false;
                }
                throw new BuildException("Can't get " + source + " to " + dest,
                        getLocation());
            }

            os = new FileOutputStream(dest);

                }
            }
            progress.endDownload();

            //if (and only if) the use file time option is set, then
            //the saved file now has its timestamp set to that of the
            //downloaded file
            if (useTimestamp)  {
                long remoteTimestamp = connection.getLastModified();
                if (verbose)  {
                    Date t = new Date(remoteTimestamp);
                    log("last modified = " + t.toString()
                        + ((remoteTimestamp == 0)
                           ? " - using current time instead"
                           : ""), logLevel);
                }
                if (remoteTimestamp != 0) {
                    FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
                }
            }
            return true;
        }

        private void updateTimeStamp() {
            long remoteTimestamp = connection.getLastModified();
            if (verbose)  {
                Date t = new Date(remoteTimestamp);
                log("last modified = " + t.toString()
                    + ((remoteTimestamp == 0)
                       ? " - using current time instead"
                       : ""), logLevel);
            }
            if (remoteTimestamp != 0) {
                FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
            }
        }
        
        /**
         * Has the download completed successfully?
         *

Return to bug 47433

Lines 18-29 Link Here

(-)src/main/org/apache/tools/ant/taskdefs/Get.java (-53 / +145 lines)
18		18
19	package org.apache.tools.ant.taskdefs;	19	package org.apache.tools.ant.taskdefs;
20		20
21	import org.apache.tools.ant.BuildException;
22	import org.apache.tools.ant.Project;
23	import org.apache.tools.ant.Task;
24	import org.apache.tools.ant.util.FileUtils;
25
26	import java.io.File;	21	import java.io.File;
		22	import java.io.FileNotFoundException;
27	import java.io.FileOutputStream;	23	import java.io.FileOutputStream;
28	import java.io.IOException;	24	import java.io.IOException;
29	import java.io.InputStream;	25	import java.io.InputStream;
Lines 33-38 Link Here
33	import java.net.URL;	29	import java.net.URL;
34	import java.net.URLConnection;	30	import java.net.URLConnection;
35	import java.util.Date;	31	import java.util.Date;
		32	import java.util.HashSet;
		33	import java.util.Set;
		34
		35	import org.apache.tools.ant.BuildException;
		36	import org.apache.tools.ant.Project;
		37	import org.apache.tools.ant.Task;
		38	import org.apache.tools.ant.util.FileUtils;
36		39
37	/**	40	/**
38	* Gets a particular file from a URL source.	41	* Gets a particular file from a URL source.
Lines 49-54 Link Here
49	private static final int DOTS_PER_LINE = 50;	52	private static final int DOTS_PER_LINE = 50;
50	private static final int BIG_BUFFER_SIZE = 100 * 1024;	53	private static final int BIG_BUFFER_SIZE = 100 * 1024;
51	private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();	54	private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();
		55	private static final int REDIRECT_LIMIT = 25;
		56
		57	private static final String HTTP = "http";
		58	private static final String HTTPS = "https";
52		59
53	private URL source; // required	60	private URL source; // required
54	private File dest; // required	61	private File dest; // required
Lines 361-366 Link Here
361	}	368	}
362		369
363	private class GetThread extends Thread {	370	private class GetThread extends Thread {
		371
364	private final boolean hasTimestamp;	372	private final boolean hasTimestamp;
365	private final long timestamp;	373	private final long timestamp;
366	private final DownloadProgress progress;	374	private final DownloadProgress progress;
Lines 371-376 Link Here
371	private BuildException exception = null;	379	private BuildException exception = null;
372	private InputStream is = null;	380	private InputStream is = null;
373	private OutputStream os = null;	381	private OutputStream os = null;
		382	private Set triedSources = new HashSet();
		383	private URLConnection connection;
374		384
375	GetThread(boolean h, long t, DownloadProgress p, int l) {	385	GetThread(boolean h, long t, DownloadProgress p, int l) {
376	hasTimestamp = h;	386	hasTimestamp = h;
Lines 390-399 Link Here
390	}	400	}
391		401
392	private boolean get() throws IOException, BuildException {	402	private boolean get() throws IOException, BuildException {
393	//set up the URL connection	403
394	URLConnection connection = source.openConnection();	404	connection = openConnection(source);
395	//modify the headers	405
396	//NB: things like user authentication could go in here too.	406	if (connection == null)
		407	{
		408	return false;
		409	}
		410
		411	boolean downloadSucceeded = downloadFile();
		412
		413	//if (and only if) the use file time option is set, then
		414	//the saved file now has its timestamp set to that of the
		415	//downloaded file
		416	if (downloadSucceeded && useTimestamp) {
		417	updateTimeStamp();
		418	}
		419
		420	return downloadSucceeded;
		421	}
		422
		423
		424	private boolean redirectionAllowed(URL aSource, URL aDest) {
		425	if (!(aSource.getProtocol().equals(aDest.getProtocol()) \|\| (HTTP
		426	.equals(aSource.getProtocol()) && HTTPS.equals(aDest
		427	.getProtocol())))) {
		428	String message = "Redirection detected from "
		429	+ aSource.getProtocol() + " to " + aDest.getProtocol()
		430	+ ". Protocol switch unsafe, not allowed.";
		431	if (ignoreErrors) {
		432	log(message, logLevel);
		433	return false;
		434	} else {
		435	throw new BuildException(message);
		436	}
		437	}
		438	if (triedSources.contains(aDest.toString())) {
		439	String message = "Cyclic redirection detected: " + aDest;
		440	if (ignoreErrors) {
		441	log(message, logLevel);
		442	return false;
		443	} else {
		444	throw new BuildException(message);
		445	}
		446	}
		447
		448	if (triedSources.size() > REDIRECT_LIMIT) {
		449	String message = "More than " + REDIRECT_LIMIT
		450	+ " times redirected, giving up";
		451	if (ignoreErrors) {
		452	log(message, logLevel);
		453	return false;
		454	} else {
		455	throw new BuildException(message);
		456	}
		457	}
		458
		459	triedSources.add(aDest);
		460	return true;
		461	}
		462
		463	private URLConnection openConnection(URL aSource) throws IOException {
		464
		465	// set up the URL connection
		466	URLConnection connection = aSource.openConnection();
		467	// modify the headers
		468	// NB: things like user authentication could go in here too.
397	if (hasTimestamp) {	469	if (hasTimestamp) {
398	connection.setIfModifiedSince(timestamp);	470	connection.setIfModifiedSince(timestamp);
399	}	471	}
Lines 401-445 Link Here
401	if (uname != null \|\| pword != null) {	473	if (uname != null \|\| pword != null) {
402	String up = uname + ":" + pword;	474	String up = uname + ":" + pword;
403	String encoding;	475	String encoding;
404	//we do not use the sun impl for portability,	476	// we do not use the sun impl for portability,
405	//and always use our own implementation for consistent	477	// and always use our own implementation for consistent
406	//testing	478	// testing
407	Base64Converter encoder = new Base64Converter();	479	Base64Converter encoder = new Base64Converter();
408	encoding = encoder.encode(up.getBytes());	480	encoding = encoder.encode(up.getBytes());
409	connection.setRequestProperty ("Authorization",	481	connection.setRequestProperty("Authorization", "Basic "
410	"Basic " + encoding);	482	+ encoding);
411	}	483	}
412		484
413	//connect to the remote site (may take some time)	485	if (connection instanceof HttpURLConnection) {
		486	((HttpURLConnection) connection)
		487	.setInstanceFollowRedirects(false);
		488	}
		489	// connect to the remote site (may take some time)
414	connection.connect();	490	connection.connect();
415	//next test for a 304 result (HTTP only)	491
		492	// First check on a 301 / 302 (moved) response
416	if (connection instanceof HttpURLConnection) {	493	if (connection instanceof HttpURLConnection) {
417	HttpURLConnection httpConnection	494	HttpURLConnection httpConnection = (HttpURLConnection) connection;
418	= (HttpURLConnection) connection;	495	// httpConnection.setInstanceFollowRedirects(false);
		496	// httpConnection.setUseCaches(false);
		497	int responseCode = httpConnection.getResponseCode();
		498	if (responseCode == HttpURLConnection.HTTP_MOVED_PERM \|\|
		499	responseCode == HttpURLConnection.HTTP_MOVED_TEMP)
		500	{
		501	String newLocation = httpConnection.getHeaderField("Location");
		502	String message = aSource
		503	+ (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
		504	: "") + " moved to " + newLocation;
		505	log(message, logLevel);
		506	URL newURL = new URL(newLocation);
		507	if (!redirectionAllowed(aSource, newURL))
		508	{
		509	return null;
		510	}
		511	return openConnection(newURL);
		512	}
		513	// next test for a 304 result (HTTP only)
419	long lastModified = httpConnection.getLastModified();	514	long lastModified = httpConnection.getLastModified();
420	if (httpConnection.getResponseCode()	515	if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
421	== HttpURLConnection.HTTP_NOT_MODIFIED	516	\|\| (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
422	\|\| (lastModified != 0 && hasTimestamp	517	// not modified so no file download. just return
423	&& timestamp >= lastModified)) {	518	// instead and trace out something so the user
424	//not modified so no file download. just return	519	// doesn't think that the download happened when it
425	//instead and trace out something so the user	520	// didn't
426	//doesn't think that the download happened when it
427	//didn't
428	log("Not modified - so not downloaded", logLevel);	521	log("Not modified - so not downloaded", logLevel);
429	return false;	522	return null;
430	}	523	}
431	// test for 401 result (HTTP only)	524	// test for 401 result (HTTP only)
432	if (httpConnection.getResponseCode()	525	if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
433	== HttpURLConnection.HTTP_UNAUTHORIZED) {
434	String message = "HTTP Authorization failure";	526	String message = "HTTP Authorization failure";
435	if (ignoreErrors) {	527	if (ignoreErrors) {
436	log(message, logLevel);	528	log(message, logLevel);
437	return false;	529	return null;
438	} else {	530	} else {
439	throw new BuildException(message);	531	throw new BuildException(message);
440	}	532	}
441	}	533	}
442
443	}	534	}
444		535
445	//REVISIT: at this point even non HTTP connections may	536	//REVISIT: at this point even non HTTP connections may
Lines 447-457 Link Here
447	//the date of the content and skip the write if it is not	538	//the date of the content and skip the write if it is not
448	//newer. Some protocols (FTP) don't include dates, of	539	//newer. Some protocols (FTP) don't include dates, of
449	//course.	540	//course.
		541	return connection;
		542	}
450		543
		544	private boolean downloadFile()
		545	throws FileNotFoundException, IOException {
451	for (int i = 0; i < NUMBER_RETRIES; i++) {	546	for (int i = 0; i < NUMBER_RETRIES; i++) {
452	//this three attempt trick is to get round quirks in different	547	// this three attempt trick is to get round quirks in different
453	//Java implementations. Some of them take a few goes to bind	548	// Java implementations. Some of them take a few goes to bind
454	//property; we ignore the first couple of such failures.	549	// property; we ignore the first couple of such failures.
455	try {	550	try {
456	is = connection.getInputStream();	551	is = connection.getInputStream();
457	break;	552	break;
Lines 464-471 Link Here
464	if (ignoreErrors) {	559	if (ignoreErrors) {
465	return false;	560	return false;
466	}	561	}
467	throw new BuildException("Can't get " + source + " to "	562	throw new BuildException("Can't get " + source + " to " + dest,
468	+ dest, getLocation());	563	getLocation());
469	}	564	}
470		565
471	os = new FileOutputStream(dest);	566	os = new FileOutputStream(dest);
Lines 491-516 Link Here
491	}	586	}
492	}	587	}
493	progress.endDownload();	588	progress.endDownload();
494
495	//if (and only if) the use file time option is set, then
496	//the saved file now has its timestamp set to that of the
497	//downloaded file
498	if (useTimestamp) {
499	long remoteTimestamp = connection.getLastModified();
500	if (verbose) {
501	Date t = new Date(remoteTimestamp);
502	log("last modified = " + t.toString()
503	+ ((remoteTimestamp == 0)
504	? " - using current time instead"
505	: ""), logLevel);
506	}
507	if (remoteTimestamp != 0) {
508	FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
509	}
510	}
511	return true;	589	return true;
512	}	590	}
513		591
		592	private void updateTimeStamp() {
		593	long remoteTimestamp = connection.getLastModified();
		594	if (verbose) {
		595	Date t = new Date(remoteTimestamp);
		596	log("last modified = " + t.toString()
		597	+ ((remoteTimestamp == 0)
		598	? " - using current time instead"
		599	: ""), logLevel);
		600	}
		601	if (remoteTimestamp != 0) {
		602	FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
		603	}
		604	}
		605
514	/**	606	/**
515	* Has the download completed successfully?	607	* Has the download completed successfully?
516	*	608	*