Attachment #19716 for bug #41854

View | Details | Raw Unified | Return to bug 41854
Collapse All | Expand All




import org.apache.xindice.xml.dom.DOMParser;
import org.apache.xindice.core.filer.BTreeFiler;
import org.apache.xindice.core.filer.BTreeCallback;
import org.apache.xindice.core.filer.BTreeException;
import org.apache.xindice.core.data.Value;
import org.apache.xindice.core.data.Key;
import org.apache.xindice.core.Database;

import org.apache.xindice.core.DBException;
import org.apache.xindice.core.indexer.Indexer;
import org.apache.xindice.util.Configuration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.File;
import java.io.IOException;

 */
public class DatabaseRebuild {

    private static final int CMD_ALL = 0;
    private static final int CMD_COPY  = 1;
    private static final int CMD_INDEX = 2;

    private static final char NOT_RAN = '_';
    private static final char SUCCESS = '*';
    private static final char ERROR = '!';
    private static int command;

    private static final Log log = LogFactory.getLog(DatabaseRebuild.class);

    public static void main(String[] args) throws Exception {
        if (args.length < 2 || args[1] == null || args[1].length() == 0 ||
            !("copy".equals(args[0]) || "index".equals(args[0]) || "rebuild".equals(args[0]))) {
            usage();
            System.exit(1);
        }

        int command;
        if ("copy".equals(args[0])) {
            command = CMD_COPY;
        } else if ("index".equals(args[0])) {
            command = CMD_INDEX;
        } else {
            command = CMD_ALL;
        }

        String dbLocation = args[1];

        File location = new File(dbLocation);
        dbLocation = location.getAbsolutePath();
        String name = location.getName();

        if ("".equals(name) || !location.exists() || !location.isDirectory()) {
            System.out.println("Database path must point to existing database directory");
            System.exit(1);
        }

        // create minimal database configuration instead of trying to locate system.xml
        String  config = "<root-collection dbroot='" + dbLocation + "/' name='" + name + "'/>";
        Database db = new Database();

        boolean status = true;
        try {
            System.out.println();
            System.out.println("CI\tCollection name");

            db.setConfig(new Configuration(DOMParser.toDocument(config)));

            if (log.isInfoEnabled()) {
                log.info("Rebuilding collections...");
            }
            status = processCollection(db, command);
        } finally {
            db.close();

            System.out.println();
            if (status) {
                System.out.println("Rebuilding database was successfull");
            } else {
                System.out.println("Rebuilding database failed. Please check logs for more detail");
            }

            System.exit(status ? 0 : 2);
        }

    }

    private static void usage() {
        System.out.println("Commands:");
        System.out.println("copy <db location> -v");
        System.out.println("index <db location> -v");
        System.out.println("rebuild <db location> -v");
    }

    private static boolean processCollection(Collection col, int command) {
        String name = col.getCanonicalName();
        boolean status;

        try {
            if (log.isInfoEnabled()) {
                log.info("Processing collection " + name);
            if (col == null) {
                System.out.println("Error fetching collection '" + colRoot + "'");
                return;
            }

            char copy = NOT_RAN;
            char index = NOT_RAN;
            switch (command) {
                case CMD_COPY:
                    status = rebuildCollection(col);
                    copy = status ? SUCCESS : ERROR;
                    break;

                case CMD_INDEX:
                    status = rebuildIndex(col);
                    index = status ? SUCCESS : ERROR;
                    break;
                default:
                    status = rebuildCollection(col);
                    copy = status ? SUCCESS : ERROR;
                    if (status) {
                        status = rebuildIndex(col);
                        index = status ? SUCCESS : ERROR;
                    }
                    break;
            }

            System.out.println(Character.toString(copy) + Character.toString(index) + "\t" + name);

            String[] colNames = col.listCollections();
            for (int i = 0; i < colNames.length; i++) {
                boolean result = processCollection(col.getCollection(colNames[i]), command);
                status = status && result;
            }
        } catch (DBException e) {
            log.error("Got an exception when processing collection " + name, e);

            return false;
        }

        return status;
    }

    private static boolean rebuildCollection(Collection col) {
        String canonicalName = col.getCanonicalName();
            case CMD_INDEX:
                rebuildIndex(col);
                break;

        if (!(col.getFiler() instanceof BTreeFiler)) {
            if (log.isInfoEnabled()) {
                log.info("Filer for collection " + canonicalName + " is not BTreeFiler. Skipping...");
            }
            return true;
        }
    }

        // close collection's filer
        try {
            col.getFiler().close();
        } catch (DBException e) {
            log.error("Could not close filer for collection " + canonicalName, e);
            return false;
        }

        // close collection and its filer
        col.close();

        System.out.println("Processing collection " + location);

        File root = new File(path + location);

        // backup
        String fileName = path + location + "/" + name;
        // FIXME What if copy fails. It's probably a better idea to first make a copy,
        //       and rename after that?
        new File(fileName + ".tbl").renameTo(new File(fileName + ".old.tbl"));

        // prepare
        BTreeCopy oldFiler = new BTreeCopy();
        oldFiler.setLocation(col.getCollectionRoot(), col.getName());
        oldFiler.setConfig(col.getFiler().getConfig());
        String oldFileName = oldFiler.getFilerFile().getAbsolutePath();
        if (!oldFiler.exists()) {
            log.error("Filer for " + oldFileName + " does not exists");
            return false;
        }

        BTreeCopy newFiler = new BTreeCopy();
        newFiler.setLocation(col.getCollectionRoot(), col.getName() + ".rebuild");
        newFiler.setConfig(col.getFiler().getConfig());
        String newFileName = newFiler.getFilerFile().getAbsolutePath();
        if (newFiler.exists()) {
            log.error("Filer for " + newFileName + " already exists");
            return false;
        }

        // copy
        if (!copy(oldFiler, newFiler, canonicalName)) {
            newFiler.deleteFile();
            return false;
        }

        oldFiler.deleteFile();
        if (!newFiler.getFilerFile().renameTo(oldFiler.getFilerFile())) {
            log.error("Could not rename successfully rebuilt file " + newFileName + " to " + oldFileName);
            return false;
        }

        try {
            col.getFiler().open();
        } catch (DBException e) {
            log.error("Could not open new file " + oldFileName, e);
            return false;
        }

        return true;
    }

    private static boolean copy(BTreeCopy oldFiler, BTreeCopy newFiler, String canonicalName) {
        try {
            newFiler.create();
            oldFiler.open();
            newFiler.open();
            oldFiler.copy(newFiler);
        } catch (Exception e) {
            log.error("Error copying collection " + canonicalName, e);
            return false;
            }
        } finally {
            try {
                oldFiler.close();
            } catch (DBException e) {
                if (log.isWarnEnabled()) log.warn(e);
            }
            try {
                newFiler.close();
            } catch (DBException e) {
                if (log.isWarnEnabled()) log.warn(e);
            }
        }

        return true;
    }

    private static boolean rebuildIndex(Collection col) {
        if (col.getFiler() == null) {
            return true;
        }

        try {
            String[] list = col.listIndexers();
            for (int i = 0; i < list.length; i++) {
                Indexer idx = col.getIndexer(list[i]);
                Configuration idxConf = idx.getConfig();
                if (log.isInfoEnabled()) {
                    log.info("Rebuilding index " + list[i] + " for collection " + col.getCanonicalName());
                }
                col.dropIndexer(idx);
                col.createIndexer(idxConf);
            }
        } catch (DBException e) {
            log.error("Could not rebuild index for collection " + col.getCanonicalName(), e);
            return false;
        }

        return true;
    }

    private static class BTreeCopy extends BTreeFiler {

            return super.readValue(pointer);
        }

        private void copy(BTreeFiler newFiler) throws IOException, BTreeException {
            query(null, new CopyCallback(this, newFiler));
        }

        private boolean deleteFile() {
            return getFile().delete();
        }

        private File getFilerFile() {
            return getFile();
        }
    }

    private static class CopyCallback implements BTreeCallback {

Return to bug 41854

Lines 22-27 Link Here

(-)java/src/org/apache/xindice/tools/DatabaseRebuild.java (-81 / +168 lines)
22	import org.apache.xindice.xml.dom.DOMParser;	22	import org.apache.xindice.xml.dom.DOMParser;
23	import org.apache.xindice.core.filer.BTreeFiler;	23	import org.apache.xindice.core.filer.BTreeFiler;
24	import org.apache.xindice.core.filer.BTreeCallback;	24	import org.apache.xindice.core.filer.BTreeCallback;
		25	import org.apache.xindice.core.filer.BTreeException;
25	import org.apache.xindice.core.data.Value;	26	import org.apache.xindice.core.data.Value;
26	import org.apache.xindice.core.data.Key;	27	import org.apache.xindice.core.data.Key;
27	import org.apache.xindice.core.Database;	28	import org.apache.xindice.core.Database;
Lines 29-35 Link Here
29	import org.apache.xindice.core.DBException;	30	import org.apache.xindice.core.DBException;
30	import org.apache.xindice.core.indexer.Indexer;	31	import org.apache.xindice.core.indexer.Indexer;
31	import org.apache.xindice.util.Configuration;	32	import org.apache.xindice.util.Configuration;
32	import org.apache.xindice.util.XindiceException;	33	import org.apache.commons.logging.Log;
		34	import org.apache.commons.logging.LogFactory;
33		35
34	import java.io.File;	36	import java.io.File;
35	import java.io.IOException;	37	import java.io.IOException;
Lines 41-192 Link Here
41	*/	43	*/
42	public class DatabaseRebuild {	44	public class DatabaseRebuild {
43		45
		46	private static final int CMD_ALL = 0;
44	private static final int CMD_COPY = 1;	47	private static final int CMD_COPY = 1;
45	private static final int CMD_INDEX = 2;	48	private static final int CMD_INDEX = 2;
46		49
47	private static Database db;	50	private static final char NOT_RAN = '_';
48	private static String path;	51	private static final char SUCCESS = '*';
49	private static boolean removeBackupFiles;	52	private static final char ERROR = '!';
50	private static int command;
51		53
		54	private static final Log log = LogFactory.getLog(DatabaseRebuild.class);
52		55
53	public static void main(String[] args) throws Exception {	56	public static void main(String[] args) throws Exception {
54	if (args.length < 2 \|\| !("copy".equals(args[0]) \|\| "index".equals(args[0])) \|\|	57	if (args.length < 2 \|\| args[1] == null \|\| args[1].length() == 0 \|\|
55	args[1] == null \|\| args[1].length() == 0) {	58	!("copy".equals(args[0]) \|\| "index".equals(args[0]) \|\| "rebuild".equals(args[0]))) {
56	usage();	59	usage();
57	return;	60	System.exit(1);
58	}	61	}
59		62
60	command = "copy".equals(args[0]) ? CMD_COPY : CMD_INDEX;	63	int command;
		64	if ("copy".equals(args[0])) {
		65	command = CMD_COPY;
		66	} else if ("index".equals(args[0])) {
		67	command = CMD_INDEX;
		68	} else {
		69	command = CMD_ALL;
		70	}
		71
61	String dbLocation = args[1];	72	String dbLocation = args[1];
62	if (args.length > 2 && args[2].equals("--remove")) {	73
63	removeBackupFiles = true;	74	File location = new File(dbLocation);
		75	dbLocation = location.getAbsolutePath();
		76	String name = location.getName();
		77
		78	if ("".equals(name) \|\| !location.exists() \|\| !location.isDirectory()) {
		79	System.out.println("Database path must point to existing database directory");
		80	System.exit(1);
64	}	81	}
65		82
66	String config = "<root-collection dbroot='" + dbLocation + "/' name='" + dbLocation + "'/>";	83	// create minimal database configuration instead of trying to locate system.xml
67	db = new Database();	84	String config = "<root-collection dbroot='" + dbLocation + "/' name='" + name + "'/>";
		85	Database db = new Database();
68		86
		87	boolean status = true;
69	try {	88	try {
		89	System.out.println();
		90	System.out.println("CI\tCollection name");
		91
70	db.setConfig(new Configuration(DOMParser.toDocument(config)));	92	db.setConfig(new Configuration(DOMParser.toDocument(config)));
71		93
72	path = db.getCollectionRoot().getPath();	94	if (log.isInfoEnabled()) {
73	processChildCollections("/");	95	log.info("Rebuilding collections...");
		96	}
		97	status = processCollection(db, command);
74	} finally {	98	} finally {
75	db.close();	99	db.close();
		100
		101	System.out.println();
		102	if (status) {
		103	System.out.println("Rebuilding database was successfull");
		104	} else {
		105	System.out.println("Rebuilding database failed. Please check logs for more detail");
		106	}
		107
		108	System.exit(status ? 0 : 2);
76	}	109	}
		110
77	}	111	}
78		112
79	private static void usage() {	113	private static void usage() {
80	System.out.println("Commands:");	114	System.out.println("Commands:");
81	System.out.println("copy <db location> [--remove]");	115	System.out.println("copy <db location> -v");
82	System.out.println("index <db location>");	116	System.out.println("index <db location> -v");
		117	System.out.println("rebuild <db location> -v");
83	}	118	}
84		119
85	private static void processChildCollections(String colRoot) {	120	private static boolean processCollection(Collection col, int command) {
86	System.out.println("Getting child collections for " + colRoot);	121	String name = col.getCanonicalName();
		122	boolean status;
		123
87	try {	124	try {
88	// Get a Collection reference	125	if (log.isInfoEnabled()) {
89	Collection col = db.getCollection(colRoot);	126	log.info("Processing collection " + name);
90	if (col == null) {
91	System.out.println("Error fetching collection '" + colRoot + "'");
92	return;
93	}	127	}
94		128
95	processCollection(col, colRoot);	129	char copy = NOT_RAN;
		130	char index = NOT_RAN;
		131	switch (command) {
		132	case CMD_COPY:
		133	status = rebuildCollection(col);
		134	copy = status ? SUCCESS : ERROR;
		135	break;
96		136
		137	case CMD_INDEX:
		138	status = rebuildIndex(col);
		139	index = status ? SUCCESS : ERROR;
		140	break;
		141	default:
		142	status = rebuildCollection(col);
		143	copy = status ? SUCCESS : ERROR;
		144	if (status) {
		145	status = rebuildIndex(col);
		146	index = status ? SUCCESS : ERROR;
		147	}
		148	break;
		149	}
		150
		151	System.out.println(Character.toString(copy) + Character.toString(index) + "\t" + name);
		152
97	String[] colNames = col.listCollections();	153	String[] colNames = col.listCollections();
98	for (int i = 0; i < colNames.length; i++) {	154	for (int i = 0; i < colNames.length; i++) {
99	processChildCollections(colRoot + colNames[i] + "/");	155	boolean result = processCollection(col.getCollection(colNames[i]), command);
		156	status = status && result;
100	}	157	}
101	} catch (Exception e) {	158	} catch (DBException e) {
102	System.out.println("Got an excefption when processing collection: " + colRoot);	159	log.error("Got an exception when processing collection " + name, e);
103	e.printStackTrace();	160
		161	return false;
104	}	162	}
		163
		164	return status;
105	}	165	}
106		166
107	private static void processCollection(Collection col, String location) throws XindiceException, IOException {	167	private static boolean rebuildCollection(Collection col) {
108	switch (command) {	168	String canonicalName = col.getCanonicalName();
109	case CMD_INDEX:
110	rebuildIndex(col);
111	break;
112		169
113	case CMD_COPY:	170	if (!(col.getFiler() instanceof BTreeFiler)) {
114	rebuildCollection(col, location, col.getName());	171	if (log.isInfoEnabled()) {
115	break;	172	log.info("Filer for collection " + canonicalName + " is not BTreeFiler. Skipping...");
		173	}
		174	return true;
116	}	175	}
117	}
118		176
119	private static void rebuildCollection(Collection col, String location, String name) throws XindiceException, IOException {	177	// close collection's filer
120	if (!(col.getFiler() instanceof BTreeFiler)) {	178	try {
121	System.out.println("Filer for collection " + location + " is not BTreeFiler. Skipping...");	179	col.getFiler().close();
122	return;	180	} catch (DBException e) {
		181	log.error("Could not close filer for collection " + canonicalName, e);
		182	return false;
123	}	183	}
124		184
125	// close collection and its filer
126	col.close();
127
128	System.out.println("Processing collection " + location);
129
130	File root = new File(path + location);
131
132	// backup
133	String fileName = path + location + "/" + name;
134	// FIXME What if copy fails. It's probably a better idea to first make a copy,
135	// and rename after that?
136	new File(fileName + ".tbl").renameTo(new File(fileName + ".old.tbl"));
137
138	// prepare	185	// prepare
139	BTreeCopy filer = new BTreeCopy();	186	BTreeCopy oldFiler = new BTreeCopy();
140	filer.setLocation(root, name + ".old");	187	oldFiler.setLocation(col.getCollectionRoot(), col.getName());
141	filer.setConfig(col.getFiler().getConfig());	188	oldFiler.setConfig(col.getFiler().getConfig());
142	if (!filer.exists()) {	189	String oldFileName = oldFiler.getFilerFile().getAbsolutePath();
143	System.out.println("Filer for " + fileName + ".old.tbl does not exists");	190	if (!oldFiler.exists()) {
144	return;	191	log.error("Filer for " + oldFileName + " does not exists");
		192	return false;
145	}	193	}
146		194
147	BTreeFiler newFiler = new BTreeFiler();	195	BTreeCopy newFiler = new BTreeCopy();
148	newFiler.setLocation(root, name);	196	newFiler.setLocation(col.getCollectionRoot(), col.getName() + ".rebuild");
149	newFiler.setConfig(col.getFiler().getConfig());	197	newFiler.setConfig(col.getFiler().getConfig());
		198	String newFileName = newFiler.getFilerFile().getAbsolutePath();
150	if (newFiler.exists()) {	199	if (newFiler.exists()) {
151	System.out.println("Filer for " + fileName + ".tbl already exists");	200	log.error("Filer for " + newFileName + " already exists");
152	return;	201	return false;
153	}	202	}
154		203
155	// copy	204	// copy
156	newFiler.create();	205	if (!copy(oldFiler, newFiler, canonicalName)) {
		206	newFiler.deleteFile();
		207	return false;
		208	}
		209
		210	oldFiler.deleteFile();
		211	if (!newFiler.getFilerFile().renameTo(oldFiler.getFilerFile())) {
		212	log.error("Could not rename successfully rebuilt file " + newFileName + " to " + oldFileName);
		213	return false;
		214	}
		215
157	try {	216	try {
158	filer.open();	217	col.getFiler().open();
		218	} catch (DBException e) {
		219	log.error("Could not open new file " + oldFileName, e);
		220	return false;
		221	}
		222
		223	return true;
		224	}
		225
		226	private static boolean copy(BTreeCopy oldFiler, BTreeCopy newFiler, String canonicalName) {
		227	try {
		228	newFiler.create();
		229	oldFiler.open();
159	newFiler.open();	230	newFiler.open();
160		231	oldFiler.copy(newFiler);
161	filer.copy(newFiler);	232	} catch (Exception e) {
162	if (removeBackupFiles) {	233	log.error("Error copying collection " + canonicalName, e);
163	filer.deleteFile();	234	return false;
164	}
165	} finally {	235	} finally {
166	try {	236	try {
167	filer.close();	237	oldFiler.close();
168	} catch (Exception e) {	238	} catch (DBException e) {
169	e.printStackTrace();	239	if (log.isWarnEnabled()) log.warn(e);
170	}	240	}
171	try {	241	try {
172	newFiler.close();	242	newFiler.close();
173	} catch (Exception e) {	243	} catch (DBException e) {
174	e.printStackTrace();	244	if (log.isWarnEnabled()) log.warn(e);
175	}	245	}
176	}	246	}
		247
		248	return true;
177	}	249	}
178		250
179	private static void rebuildIndex(Collection col) throws DBException {	251	private static boolean rebuildIndex(Collection col) {
180	if (col.getFiler() != null) {	252	if (col.getFiler() == null) {
		253	return true;
		254	}
		255
		256	try {
181	String[] list = col.listIndexers();	257	String[] list = col.listIndexers();
182	for (int i = 0; i < list.length; i++) {	258	for (int i = 0; i < list.length; i++) {
183	Indexer idx = col.getIndexer(list[i]);	259	Indexer idx = col.getIndexer(list[i]);
184	Configuration idxConf = idx.getConfig();	260	Configuration idxConf = idx.getConfig();
185	System.out.println("Rebuilding index " + list[i] + " for collection " + col.getName());	261	if (log.isInfoEnabled()) {
		262	log.info("Rebuilding index " + list[i] + " for collection " + col.getCanonicalName());
		263	}
186	col.dropIndexer(idx);	264	col.dropIndexer(idx);
187	col.createIndexer(idxConf);	265	col.createIndexer(idxConf);
188	}	266	}
		267	} catch (DBException e) {
		268	log.error("Could not rebuild index for collection " + col.getCanonicalName(), e);
		269	return false;
189	}	270	}
		271
		272	return true;
190	}	273	}
191		274
192	private static class BTreeCopy extends BTreeFiler {	275	private static class BTreeCopy extends BTreeFiler {
Lines 194-206 Link Here
194	return super.readValue(pointer);	277	return super.readValue(pointer);
195	}	278	}
196		279
197	private void copy(BTreeFiler newFiler) throws XindiceException, IOException {	280	private void copy(BTreeFiler newFiler) throws IOException, BTreeException {
198	query(null, new CopyCallback(this, newFiler));	281	query(null, new CopyCallback(this, newFiler));
199	}	282	}
200		283
201	private boolean deleteFile() {	284	private boolean deleteFile() {
202	return getFile().delete();	285	return getFile().delete();
203	}	286	}
		287
		288	private File getFilerFile() {
		289	return getFile();
		290	}
204	}	291	}
205		292
206	private static class CopyCallback implements BTreeCallback {	293	private static class CopyCallback implements BTreeCallback {