Bug 15241 - Enhance get to process a list of resources from a file
Summary: Enhance get to process a list of resources from a file
Status: RESOLVED FIXED
Alias: None
Product: Ant
Classification: Unclassified
Component: Core tasks (show other bugs)
Version: 1.5.1
Hardware: Sun All
: P3 enhancement with 1 vote (vote)
Target Milestone: 1.8.0
Assignee: Ant Notifications List
URL:
Keywords: PatchAvailable
Depends on:
Blocks:
 
Reported: 2002-12-10 16:24 UTC by Robert Dobbins
Modified: 2009-09-29 08:04 UTC (History)
0 users



Attachments
Patch for enhancement to Get task (17.77 KB, patch)
2002-12-10 16:26 UTC, Robert Dobbins
Details | Diff

Note You need to log in before you can comment on or make changes to this bug.
Description Robert Dobbins 2002-12-10 16:24:08 UTC
Added the ability to specify a file parameter that contains a list of resources 
to get from a URL.  The dest must be a directory and the files are copied 
without renaming.

--- Get.java.Orig       Tue Dec  3 12:47:26 2002
+++ Get.java    Tue Dec  3 12:48:19 2002
@@ -56,17 +56,22 @@
 
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.InputStream;
-import java.net.HttpURLConnection;
+import java.io.IOException;
+import java.io.FileReader;
+import java.io.BufferedReader;
 import java.net.URL;
+import java.net.MalformedURLException;
 import java.net.URLConnection;
+import java.net.HttpURLConnection;
 import java.util.Date;
 import org.apache.tools.ant.BuildException;
-import org.apache.tools.ant.Project;
 import org.apache.tools.ant.Task;
+import org.apache.tools.ant.Project;
 import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.StringUtils;
 import org.apache.tools.ant.util.JavaEnvUtils;
+import java.util.ArrayList;
 
 /**
  * Gets a particular file from a URL source.
@@ -84,11 +89,13 @@
 public class Get extends Task {
     private URL source; // required
     private File dest; // required
+    private File file; // optional
     private boolean verbose = false;
     private boolean useTimestamp = false; //off by default
     private boolean ignoreErrors = false;
     private String uname = null;
     private String pword = null;
+    private int i;
 
 
     /**
@@ -97,174 +104,57 @@
      * @exception BuildException Thrown in unrecoverable error.
      */
     public void execute() throws BuildException {
+       ArrayList modules = null;
+     
         if (source == null) {
-            throw new BuildException("src attribute is required", getLocation
());
+            throw new BuildException("src attribute is required", location);
         }
 
         if (dest == null) {
-            throw new BuildException("dest attribute is required", getLocation
());
-        }
-
-        if (dest.exists() && dest.isDirectory()) {
-            throw new BuildException("The specified destination is a 
directory",
-                                     getLocation());
+            throw new BuildException("dest attribute is required", location);
         }
 
         if (dest.exists() && !dest.canWrite()) {
             throw new BuildException("Can't write to " + dest.getAbsolutePath
(),
-                                     getLocation());
+                                     location);
         }
 
-        try {
-
-            log("Getting: " + source);
-
-            //set the timestamp to the file date.
-            long timestamp = 0;
-
-            boolean hasTimestamp = false;
-            if (useTimestamp && dest.exists()) {
-                timestamp = dest.lastModified();
-                if (verbose) {
-                    Date t = new Date(timestamp);
-                    log("local file date : " + t.toString());
-                }
-
-                hasTimestamp = true;
-            }
-
-            //set up the URL connection
-            URLConnection connection = source.openConnection();
-            //modify the headers
-            //NB: things like user authentication could go in here too.
-            if (useTimestamp && hasTimestamp) {
-                connection.setIfModifiedSince(timestamp);
-            }
-            // prepare Java 1.1 style credentials
-            if (uname != null || pword != null) {
-                String up = uname + ":" + pword;
-                String encoding;
-                // check to see if sun's Base64 encoder is available.
-                try {
-                    sun.misc.BASE64Encoder encoder =
-                        (sun.misc.BASE64Encoder)
-                        Class.forName("sun.misc.BASE64Encoder").newInstance();
-                    encoding = encoder.encode (up.getBytes());
-
-                } catch (Exception ex) { // sun's base64 encoder isn't 
available
-                    Base64Converter encoder = new Base64Converter();
-                    encoding = encoder.encode(up.getBytes());
-                }
-                connection.setRequestProperty ("Authorization",
-                                               "Basic " + encoding);
-            }
-
-            //connect to the remote site (may take some time)
-            connection.connect();
-            //next test for a 304 result (HTTP only)
-            if (connection instanceof HttpURLConnection) {
-                HttpURLConnection httpConnection
-                    = (HttpURLConnection) connection;
-                if (httpConnection.getResponseCode()
-                    == HttpURLConnection.HTTP_NOT_MODIFIED)  {
-                    //not modified so no file download. just return
-                    //instead and trace out something so the user
-                    //doesn't think that the download happened when it
-                    //didnt
-                    log("Not modified - so not downloaded");
-                    return;
-                }
-                // test for 401 result (HTTP only)
-                if (httpConnection.getResponseCode()
-                    == HttpURLConnection.HTTP_UNAUTHORIZED)  {
-                    String message="HTTP Authorization failure";
-                    if(ignoreErrors) {
-                        log(message,Project.MSG_WARN);
-                        return;
-                    } else {
-                        throw new BuildException(message);
-                    }
-                }
-
+        /**
+         * Check for file designation, if present dest must be a dir. and 
source must not contain a filename 
+         */
+        if (file != null){
+            if (dest.exists() && !dest.isDirectory()) {
+                throw new BuildException("The specified destination must be a 
directory",
+                                         location);
+            }
+            if (!dest.exists()){
+               if(!dest.mkdirs()) 
+                       throw new BuildException("Could not make new 
directory.");
+            }
+            modules = loadFile(file);
+
+            FileUtils fu = FileUtils.newFileUtils();
+            StringUtils su = new StringUtils();
+   
+            try{
+                source = new URL(su.replace(source.toString() + "/","//","/"));
+                source = new URL(su.replace(source.toString(),":/","://"));
+                for (i = 0; i < modules.size(); i++){
+               doWork(new URL(source.toString() + modules.get(i)), fu.normalize
(dest.toString() + "/" + modules.get(i)));
+                }
+            }
+            catch(MalformedURLException mfURL){
+               throw new BuildException("mfURL " + mfURL.toString());
+            }
+            
+        } else {
+            if (dest.exists() && dest.isDirectory()) {
+                throw new BuildException("The specified destination is a 
directory",
+                                         location);
             }
-
-            //REVISIT: at this point even non HTTP connections may
-            //support the if-modified-since behaviour -we just check
-            //the date of the content and skip the write if it is not
-            //newer. Some protocols (FTP) dont include dates, of
-            //course.
-
-            InputStream is = null;
-            for (int i = 0; i < 3 ; i++) {
-                try {
-                    is = connection.getInputStream();
-                    break;
-                } catch (IOException ex) {
-                    log("Error opening connection " + ex);
-                }
-            }
-            if (is == null) {
-                log("Can't get " + source + " to " + dest);
-                if (ignoreErrors) {
-                    return;
-                }
-                throw new BuildException("Can't get " + source + " to " + dest,
-                                         getLocation());
-            }
-
-            FileOutputStream fos = new FileOutputStream(dest);
-            boolean finished = false;
-            try {
-                byte[] buffer = new byte[100 * 1024];
-                int length;
-
-                while ((length = is.read(buffer)) >= 0) {
-                    fos.write(buffer, 0, length);
-                    if (verbose) {
-                        System.out.print(".");
-                    }
-                }
-                if (verbose) {
-                    System.out.println();
-                }
-                finished = true;
-            } finally {
-                if (fos != null) {
-                    fos.close();
-                }
-                is.close();
-                // we have started to (over)write dest, but failed.
-                // Try to delete the garbage we'd otherwise leave
-                // behind.
-                if (!finished) {
-                    dest.delete();
-                }
-            }
-
-            //if (and only if) the use file time option is set, then
-            //the saved file now has its timestamp set to that of the
-            //downloaded file
-            if (useTimestamp)  {
-                long remoteTimestamp = connection.getLastModified();
-                if (verbose)  {
-                    Date t = new Date(remoteTimestamp);
-                    log("last modified = " + t.toString()
-                        + ((remoteTimestamp == 0)
-                          ? " - using current time instead"
-                          : ""));
-                }
-                if (remoteTimestamp != 0) {
-                    FileUtils.newFileUtils()
-                        .setFileLastModified(dest, remoteTimestamp);
-                }
-            }
-        } catch (IOException ioe) {
-            log("Error getting " + source + " to " + dest);
-            if (ignoreErrors) {
-                return;
-            }
-            throw new BuildException(ioe, getLocation());
+            doWork(source, dest);
         }
+
     }
 
     /**
@@ -286,6 +176,15 @@
     }
 
     /**
+     * Where to copy the source file.
+     *
+     * @param dest Path to file.
+     */
+    public void setFile(File file) {
+        this.file = file;
+    }
+
+    /**
      * If true, show verbose progress information.
      *
      * @param v if "true" then be verbose
@@ -428,4 +327,195 @@
             return new String(out);
         }
      }
+         /**
+     * load properties from a file
+     * @param file file to load
+     */
+    protected ArrayList loadFile(File file) throws BuildException {
+       ArrayList _al = null;
+    
+        log("Loading " + file.getAbsolutePath(), Project.MSG_VERBOSE);
+        try {
+            if (file.exists()) {
+                BufferedReader in = new BufferedReader(new FileReader(file));
+                //Create the list to act on.
+                _al = new ArrayList();
+                try {
+                   String _st = in.readLine();
+                       while (_st != null){
+                          if (!_st.trim().equals("") && 
+                              !_st.trim().substring(0,1).equals("#")){
+                                   _al.add(_st.trim());
+                        }
+                        _st = in.readLine();
+                       }
+                } finally {
+                    if (in != null) {
+                        in.close();
+                    }
+                }
+            } else {
+                log("Unable to find file: " + file.getAbsolutePath(),
+                    Project.MSG_VERBOSE);
+                throw new BuildException("Unable to find file: " + 
file.getAbsolutePath());
+            }
+            return _al;
+        } catch (IOException ex) {
+            throw new BuildException(ex, getLocation());
+        }
+    }
+    private void doWork(URL source, File dest){
+               try {
+
+            log("Getting: " + source);
+
+            //set the timestamp to the file date.
+            long timestamp = 0;
+
+            boolean hasTimestamp = false;
+            if (useTimestamp && dest.exists()) {
+                timestamp = dest.lastModified();
+                if (verbose) {
+                    Date t = new Date(timestamp);
+                    log("local file date : " + t.toString());
+                }
+
+                hasTimestamp = true;
+            }
+
+            //set up the URL connection
+            URLConnection connection = source.openConnection();
+            //modify the headers
+            //NB: things like user authentication could go in here too.
+            if (useTimestamp && hasTimestamp) {
+                connection.setIfModifiedSince(timestamp);
+            }
+            // prepare Java 1.1 style credentials
+            if (uname != null || pword != null) {
+                String up = uname + ":" + pword;
+                String encoding;
+                // check to see if sun's Base64 encoder is available.
+                try {
+                    sun.misc.BASE64Encoder encoder =
+                        (sun.misc.BASE64Encoder)
+                        Class.forName("sun.misc.BASE64Encoder").newInstance();
+                    encoding = encoder.encode (up.getBytes());
+
+                } catch (Exception ex) { // sun's base64 encoder isn't 
available
+                    Base64Converter encoder = new Base64Converter();
+                    encoding = encoder.encode(up.getBytes());
+                }
+                connection.setRequestProperty ("Authorization",
+                                               "Basic " + encoding);
+            }
+
+            //connect to the remote site (may take some time)
+            connection.connect();
+            //next test for a 304 result (HTTP only)
+            if (connection instanceof HttpURLConnection) {
+                HttpURLConnection httpConnection
+                    = (HttpURLConnection) connection;
+                if (httpConnection.getResponseCode()
+                    == HttpURLConnection.HTTP_NOT_MODIFIED)  {
+                    //not modified so no file download. just return
+                    //instead and trace out something so the user
+                    //doesn't think that the download happened when it
+                    //didnt
+                    log("Not modified - so not downloaded");
+                    return;
+                }
+                // test for 401 result (HTTP only)
+                if (httpConnection.getResponseCode()
+                    == HttpURLConnection.HTTP_UNAUTHORIZED)  {
+                    String message="HTTP Authorization failure";
+                    if(ignoreErrors) {
+                        log(message,Project.MSG_WARN);
+                        return;
+                    } else {
+                        throw new BuildException(message);
+                    }
+                }
+
+            }
+
+            //REVISIT: at this point even non HTTP connections may
+            //support the if-modified-since behaviour -we just check
+            //the date of the content and skip the write if it is not
+            //newer. Some protocols (FTP) dont include dates, of
+            //course.
+
+            InputStream is = null;
+            for (int i = 0; i < 3 ; i++) {
+                try {
+                    is = connection.getInputStream();
+                    break;
+                } catch (IOException ex) {
+                    log("Error opening connection " + ex);
+                }
+            }
+            if (is == null) {
+                log("Can't get " + source + " to " + dest);
+                if (ignoreErrors) {
+                    return;
+                }
+                throw new BuildException("Can't get " + source + " to " + dest,
+                                          location);
+            }
+
+            FileOutputStream fos = new FileOutputStream(dest);
+            boolean finished = false;
+            try {
+                byte[] buffer = new byte[100 * 1024];
+                int length;
+
+                while ((length = is.read(buffer)) >= 0) {
+                    fos.write(buffer, 0, length);
+                    if (verbose) {
+                        System.out.print(".");
+                    }
+                }
+                if (verbose) {
+                    System.out.println();
+                }
+                finished = true;
+            } finally {
+                if (fos != null) {
+                    fos.close();
+                }
+                is.close();
+                // we have started to (over)write dest, but failed.
+                // Try to delete the garbage we'd otherwise leave
+                // behind.
+                if (!finished) {
+                    dest.delete();
+                }
+            }
+
+            //if (and only if) the use file time option is set, then
+            //the saved file now has its timestamp set to that of the
+            //downloaded file
+            if (useTimestamp)  {
+                long remoteTimestamp = connection.getLastModified();
+                if (verbose)  {
+                    Date t = new Date(remoteTimestamp);
+                    log("last modified = " + t.toString()
+                        + ((remoteTimestamp == 0)
+                          ? " - using current time instead"
+                          : ""));
+                }
+                if (remoteTimestamp != 0) {
+                    FileUtils.newFileUtils()
+                        .setFileLastModified(dest, remoteTimestamp);
+                }
+            }
+        } catch (IOException ioe) {
+            log("Error getting " + source + " to " + dest);
+            if (ignoreErrors) {
+                return;
+            }
+            throw new BuildException(ioe, location);
+        }
+    
+    }
+     
 }
Comment 1 Robert Dobbins 2002-12-10 16:26:36 UTC
Created attachment 4115 [details]
Patch for enhancement to Get task
Comment 2 Stefan Bodewig 2009-09-29 06:48:21 UTC
<get> supports getting multiple resources since svn revision 818129 (the obvious mapper error has been fixed in svn revision 818135 ;-)

If you combine this with the skipexisting attribute, your usecase will (finally) be addressed by Ant 1.8.0.

There will likely be a resource collection that will allow reading a list of resources from a file as well.
Comment 3 Matt Benson 2009-09-29 08:04:02 UTC
(In reply to comment #2)
> <get> supports getting multiple resources since svn revision 818129 (the
> obvious mapper error has been fixed in svn revision 818135 ;-)
> 
> If you combine this with the skipexisting attribute, your usecase will
> (finally) be addressed by Ant 1.8.0.
> 
> There will likely be a resource collection that will allow reading a list of
> resources from a file as well.

Surely you mean reading a list of resources from another resource?  ;)