/**
 * January 2007
 * @author Mike Walker
 *
 * Please see the user's guide for a full description of capabilties, etc.
 *
 * Description/Assumptions:
 * 1. Table's name in source defines the base DN (or context) for the search.
 * Example: Table.NameInSource=ou=people,dc=gene,dc=com
 * [Optional] The table's name in source can also define a search scope. Append
 * a "?" character as a delimiter to the base DN, and add the search scope string.
 * The following scopes are available:
 * SUBTREE_SCOPE
 * ONELEVEL_SCOPE
 * OBJECT_SCOPE
 * [Default] LDAPConnectorConstants.ldapDefaultSearchScope
 * is the default scope used, if no scope is defined (currently, ONELEVEL_SCOPE).
 *
 * 2. Column's name in source defines the LDAP attribute name.
 * [Default] If no name in source is defined, then we attempt to use the column name
 * as the LDAP attribute name.
 *
 *
 * TODO: Implement paged searches -- the LDAP server must support VirtualListViews.
 * TODO: Implement cancel.
 * TODO: Add Sun/Netscape implementation, AD/OpenLDAP implementation.
 *
 *
 * Note:
 * Greater than is treated as >=
 * Less-than is treater as <=
 * If an LDAP entry has more than one entry for an attribute of interest (e.g. a select item), we only return the
 * first occurrance. The first occurance is not predictably the same each time, either, according to the LDAP spec.
 * If an attribute is not present, we return the empty string. Arguably, we could throw an exception.
 *
 * Sun LDAP won't support Sort Orders for very large datasets. So, we've set the sorting to NONCRITICAL, which
 * allows Sun to ignore the sort order. This will result in the results to come back as unsorted, without any error.
 *
 * Removed support for ORDER BY for two reasons:
 * 1: LDAP appears to have a limit to the number of records that
 * can be server-side sorted. When the limit is reached, two things can happen:
 * a. If sortControl is set to CRITICAL, then the search fails.
 * b. If sortControl is NONCRITICAL, then the search returns, unsorted.
 * We'd like to support ORDER BY, no matter how large the size, so we turn it off,
 * and allow MetaMatrix to do it for us.
 * 2: Supporting ORDER BY appears to negatively effect the query plan
 * when cost analysis is used. We stop using dependent queries, and start
 * using inner joins.
 *
 */

package com.metamatrix.services.LDAPConnectorv2;

import com.metamatrix.data.api.Batch;
import com.metamatrix.data.basic.BasicBatch;
import com.metamatrix.data.api.ConnectorCapabilities;
import com.metamatrix.data.api.ExecutionContext;
import com.metamatrix.data.api.SynchQueryExecution;
import com.metamatrix.data.exception.ConnectorException;
import com.metamatrix.data.language.*;
import com.metamatrix.data.metadata.runtime.*;
import com.metamatrix.data.api.ConnectorLogger;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.io.IOException;

import javax.naming.*;
import javax.naming.directory.*;
import javax.naming.ldap.*;
import java.text.SimpleDateFormat;
import java.text.ParseException;
import java.sql.Timestamp;
//import com.netscape.jndi.ldap.controls.*;
// import netscape.ldap.controls.*;

/**
 * Responsible for executing an LDAP search corresponding to a read-only "select" query from MMX.
 */
public class LDAPSyncQueryExecution implements SynchQueryExecution {
	private final int defaultSearchScope = SearchControls.SUBTREE_SCOPE;
	private ConnectorLogger logger;
	private LDAPSearchDetails searchDetails;
	private ExecutionContext ctx;
	private RuntimeMetadata rm;
	private int maxBatchSize;
	private InitialLdapContext initialLdapContext;
	private LdapContext ldapCtx;
	private NamingEnumeration searchEnumeration;
	private byte[] cookie;
	private int rowsProcessed = 0;
	private IQueryToLdapSearchParser parser;
	// This delimiter can be eventually defined in 
	private String delimiter = "?";

	public LDAPSyncQueryExecution(int executionMode, ExecutionContext ctx,
			RuntimeMetadata rm, ConnectorLogger logger,
			InitialLdapContext ldapCtx) throws ConnectorException {
		if (executionMode != ConnectorCapabilities.EXECUTION_MODE.SYNCH_QUERY) {
			throw new ConnectorException(
					"Synchronous query was called by connector, but request does not match. Cannot execute request.");
		}
		this.ctx = ctx;
		this.rm = rm;
		this.logger = logger;
		this.initialLdapContext = ldapCtx;
	}

	public void execute(IQuery query, int maxBatchSize)
			throws ConnectorException {
		this.maxBatchSize = maxBatchSize;

		// Parse the IQuery, and translate it into an appropriate LDAP search.
		this.parser = new IQueryToLdapSearchParser(logger, rm);
		searchDetails = parser.translateSQLQueryToLDAPSearch(query);

		// Create and configure the new search context.
		createSearchContext();
		SearchControls ctrls = setSearchControls();
		if (LDAPConnectorConstants.isSunLDAPServer) {
			setSunRequestControls(maxBatchSize);
		} else {
			setStandardRequestControls();
		}
		if (false) {
			// TODO: Handle AD paged results case here.
		}
		// Execute the search.
		executeSearch(ctrls);
	}

	private void setStandardRequestControls() throws ConnectorException {
		Control[] sortCtrl = new Control[1];
		SortKey[] keys = searchDetails.getSortKeys();
		if (keys != null) {
			try {
				sortCtrl[0] = new SortControl(keys, Control.NONCRITICAL);
				this.ldapCtx.setRequestControls(sortCtrl);
				logger.logTrace("Sort ordering was requested, and sort control was created successfully.");
			} catch (NamingException ne) {
				throw new ConnectorException(ne,
						"Failed to set standard sort controls. "
						+ "Please verify that the server supports sorting, and that the bind user has permission to use sort controls.");
			} catch(IOException e) {
				throw new ConnectorException(e,
						"Failed to set standard sort controls. "
						+ "Please verify that the server supports sorting, and that the bind user has permission to use sort controls.");
			}
		}


	}

	/** Perform a lookup against the initial LDAP context, which
	 * sets the context to something appropriate for the search that is about to occur.
	 *
	 */
	private void createSearchContext() throws ConnectorException {
		try {
			ldapCtx = (LdapContext) this.initialLdapContext.lookup(searchDetails.getContextName());
		} catch (NamingException ne) {
			if (searchDetails.getContextName() != null) {
				logger.logError("Attempted to search context: "
						+ searchDetails.getContextName());
			}
			throw new ConnectorException(ne, "Failed to create Ldap search context from the specified context name. " +
					"Check the table/group name in source to ensure the context exists.");
		}
	}

	/** For Sun/Netscape Ldap implementations, we will use their virtual list
	 * control and sort controls to implement sorting and paging. That's becaue the JNDI
	 * paging controls are not supported.
	 */
	private void setSunRequestControls(int maxBatchSize) {
		/*
		 * Control[] sunCtrls = new Control[2];
		 *  // Determine the sort key LdapSortKey[] ldapKeys =
		 * searchDetails.getNetscapeSortKeys(); if(ldapKeys == null) { ldapKeys =
		 * new LdapSortKey[1]; ldapKeys[0] = new
		 * LdapSortKey(LDAPConnectorConstants.ldapDefaultSortName,
		 * LDAPConnectorConstants.ldapDefaultIsAscending); } sunCtrls[0] = new
		 * LdapSortControl(ldapKeys, true);
		 *
		 * sunCtrls[1] = new LdapVirtualListControl(1, 0, maxBatchSize-1, 0);
		 * try { this.ldapCtx.setRequestControls(sunCtrls); }
		 * catch(NamingException ne) { logger.logError("Failed to set
		 * Sun-specific request controls. " + "Please ensure that the LDAP
		 * server supports server-side sorting and virtual list controls.");
		 * ne.printStackTrace(); }
		 */
	}

	/*
	 * This example should be thrown into a separate execution class that
	 * extends support for Sun's VLVs.
	 */
	private void resetVirtualListRequest() {
		// use setRange here to bump the range up to the next batch size.
	}

	private SearchControls setSearchControls() throws ConnectorException {
		SearchControls ctrls = new SearchControls();
		//ArrayList modelAttrList = searchDetails.getAttributeList();
		ArrayList modelAttrList = searchDetails.getElementList();
		String[] attrs = new String[modelAttrList.size()];
		Iterator itr = modelAttrList.iterator();
		int i = 0;
		while(itr.hasNext()) {
			attrs[i] = (parser.getNameFromElement((Element)itr.next()));
			//attrs[i] = (((Attribute)itr.next()).getID();
			//logger.logTrace("Adding attribute named " + attrs[i] + " to the search list.");
			i++;
		}

		if(attrs == null) {
			throw new ConnectorException("Failed to configure attributes properly.");
		}

		ctrls.setSearchScope(searchDetails.getSearchScope());
		ctrls.setReturningAttributes(attrs);

		long limit = searchDetails.getCountLimit();
		if(limit != -1) {
			ctrls.setCountLimit(limit);
		}
		return ctrls;
	}

	/**
	 * Perform the LDAP search against the subcontext, using the filter and
	 * search controls appropriate to the query and model metadata.
	 */
	private void executeSearch(SearchControls ctrls) throws ConnectorException {
		rowsProcessed = 0;
		String ctxName = searchDetails.getContextName();
		String filter = searchDetails.getContextFilter();
		if (ctxName == null || filter == null || ctrls == null) {
			logger.logError("Search context, filter, or controls were null. Cannot execute search.");
		}
		try {
			// TODO: Remove
			logger.logTrace("DEBUG: executing search");
			searchEnumeration = this.ldapCtx.search("", filter, ctrls);
		} catch (NamingException ne) {
			logger.logError("LDAP search failed. Attempted to search context "
					+ ctxName + " using filter " + filter);
			throw new ConnectorException(ne, "Execute search failed. Please check logs for search details.");
		} catch(Exception e) {
			logger.logError("LDAP search failed. Attempted to search context "
					+ ctxName + " using filter " + filter);
			throw new ConnectorException(e, "Execute search failed. Please check logs for search details.");
		}
	}

	/**
	 * Not implemented or supported for standard JNDI searches.
	 * (non-Javadoc)
	 * @see com.metamatrix.data.api.Execution#cancel()
	 */
	public void cancel() throws ConnectorException {

		// TODO This could acutally be implemented if we are using paged
		// results...

	}

	public void close() throws ConnectorException {
		try {
			if(ldapCtx != null) {
				ldapCtx.close();
			}
		} catch (NamingException ne) {
			logger.logError("Ldap error occurred during attempt to close context.");
			ne.printStackTrace();
		}
	}
	/**
	 * Fetch the next batch of data from the LDAP searchEnumerationr result.
	 */
	public Batch nextBatch() throws ConnectorException {
		// TODO: Remove
		logger.logTrace("DEBUG: process next batch");
		Batch batch = new BasicBatch();
		int curBatchSize = 0;
		try {
			// The search has been executed, so process up to one batch of
			// results.
			while (searchEnumeration != null && searchEnumeration.hasMore()
					&& curBatchSize < maxBatchSize)
			{
				SearchResult result = (SearchResult) searchEnumeration.next();
				addRowToBatch(batch, result);
				curBatchSize++;
			}
			rowsProcessed += curBatchSize;

			if (!searchEnumeration.hasMore()) {
				if(batch==null) {
					throw new ConnectorException("Batch became null. DEBUG");
				}
				batch.setLast();
			} else {
				/*
				 * if(LDAPConnectorConstants.isSunLDAPServer) { Control[]
				 * responseCtrls = this.ldapCtx.getResponseControls(); for(int
				 * i=0; i<responseCtrls.length; i++) { if(responseCtrls[i]
				 * instanceof LdapVirtualListResponseControl) { cookie =
				 * ((LdapVirtualListResponseControl)responseCtrls[i]).getContext(); } }
				 * resetVirtualListRequest();
				 */

			}

		} catch (SizeLimitExceededException e) {
			logger.logWarning("Search results exceeded size limit. Results may be incomplete.");
		}catch (NamingException ne) {
			logger.logError("Ldap error while processing next batch of results.");
			ne.printStackTrace();
		}
		return batch;
	}

	private void addRowToBatch(Batch batch, SearchResult result) throws ConnectorException, NamingException {
		Attributes attrs = result.getAttributes();
		//String[] attrNames = searchDetails.getAttributes();
		//ArrayList attributeList = searchDetails.getAttributeList();
		ArrayList attributeList = searchDetails.getElementList();
		List row = new ArrayList();

		//NamingEnumeration ne = attrs.getIDs();
		//while(ne.hasMore()) {
			//logger.logTrace("Received attribute named: " + (String)ne.next());
		//}

		if (attrs != null) {
			Iterator itr = attributeList.iterator();
			while(itr.hasNext()) {
				addResultToRow((Element)itr.next(), attrs, row);
			}
			//for (int i = 0; i < attrNames.length; i++) {
				//addResultToRow(attrNames[i], attrs, row);
			//}
		}
		batch.addRow(row);
	}

	//private void addResultToRow(String attrName, Attributes attrs, List row) throws ConnectorException {
	private void addResultToRow(Element modelElement, Attributes attrs, List row) throws ConnectorException, NamingException {
		String strResult;
		//String modelAttrName = modelAttr.getID();
		String modelAttrName = parser.getNameFromElement(modelElement);
		//Class modelAttrClass = (Class)modelAttr.get();
		Class modelAttrClass = (Class)modelElement.getJavaType();
		
		// mpw 5/09
		String multivalAttr = (String)modelElement.getDefaultValue();
		if(multivalAttr==null) {
			multivalAttr = "";
		}
		// end mpw 5/09
		
		if(modelAttrName == null) {
			throw new ConnectorException("Received null attribute name for a select symbol. Please check name in source for each column.");
		}
		//logger.logTrace("Looking for attribute named: " + modelAttrName);
		Attribute resultAttr = attrs.get(modelAttrName);

		// If the attribute is not present, we return NULL.
		if(resultAttr == null) {
			// MPW - 2-20-07 - Changed from returning empty string to returning null.
			//row.add("");
			//logger.logTrace("Did not find a match for attribute named: " + modelAttrName);
			row.add(null);
			return;
		} else {
			// TODO: Currently, if an LDAP entry contains more than one matching
			// attribute, we only return the first.
			// Since attribute order is not guaranteed, this means that we may not
			// always return the exact same information.
			// Putting multi-valued attributes into a single row (or multiple rows) requires
			// some design decisions.
			
			// mpw 5/09
			// MPW 2009-05-15: added support for multi-value attributes for String type only
			// Must specify "multivalued-concat" in "DefaultValue" for a String type in source model.
			try {
				// LDAP always returns either strings or byte[]. We will only deal with strings.
				
				//	get this value, but we might get all the values later on if it is multi-valued
				strResult = (String)resultAttr.get();

				// MPW - 3.9.07 - Also return NULL when attribute is unset or empty string.
				// There is no way to differentiate between being unset and being the empty string.
				if(strResult.equals("")) {
					strResult = null;
				}
			} catch (NamingException ne) {
				logger.logWarning("Failed to fetch attribute value for " + modelAttrName);
				throw new ConnectorException(ne, "Failed to fetch attribute value for attribute " + modelAttrName + ". Rowset cannot be constructed from incomplete LDAP results.");
			}

			// MPW: 3-11-07: Added support for java.lang.Integer conversion.
			try {
				if(modelAttrClass.equals(Class.forName(Integer.class.getName()))) {
					try {
						//	Throw an exception if class cast fails.
						if(strResult != null) {
							Integer intResult = new Integer(strResult);
							row.add(intResult);
						} else {
							row.add(null);
						}
					} catch(NumberFormatException nfe) {
						logger.logWarning(nfe.getMessage() + ":Element " + modelAttrName + " is typed as Integer, " +
								"but it's value (" + strResult + ") cannot be converted from string " +
								"to Integer. Defaulted to 0");
						row.add(new Integer(0));
					}
				// java.lang.String
				} else if(modelAttrClass.equals(Class.forName(String.class.getName()))) {
					// mpw 5/09
					// Order the multi-valued attrs alphabetically before creating a single string, 
					// using the delimiter to separate each token
					String multivalResult = "";
					if(multivalAttr != null && multivalAttr.equalsIgnoreCase("multivalued-concat")) {
						ArrayList<String> multivalList = new ArrayList<String>();
						NamingEnumeration attrNE = resultAttr.getAll();
						while(attrNE.hasMore()) {
							multivalList.add((String)attrNE.next());
						}
						Collections.sort(multivalList);

						// Ends up being no faster.
						//Pattern pattern = Pattern.compile(", ");
						//Matcher matcher = pattern.matcher(multivalList.toString());
						//String output = matcher.replaceAll(delimiter);

						// Use StringBuffer -- much more efficient for building these strings!
						StringBuffer multivalSB = new StringBuffer(multivalList.toString().length());
						Iterator itr = multivalList.iterator();
						int i=0;
						while(itr.hasNext()) {
							if(i==0) {
								//multivalResult = (String)itr.next();
								multivalSB.append((String)itr.next());
							}
							else {
								//multivalResult = multivalResult + delimiter + (String)itr.next();
								multivalSB.append(delimiter);
								multivalSB.append((String)itr.next());
							}
							i++;
						}
						row.add(multivalSB.toString());
					} else {
						row.add(strResult);
					}
					// end mpw 5/09
				// java.sql.Timestamp
				} else if(modelAttrClass.equals(Class.forName(java.sql.Timestamp.class.getName()))) {
					Properties p = modelElement.getProperties();
					//Enumeration e = p.propertyNames();
					//while(e.hasMoreElements()) {
						//logger.logTrace("Property name of a timestamp column: " + e.nextElement());
					//}
					String timestampFormat = p.getProperty("Format");
					SimpleDateFormat dateFormat;
					if(timestampFormat == null) {
						timestampFormat = LDAPConnectorConstants.ldapTimestampFormat;

					}
					dateFormat = new SimpleDateFormat(timestampFormat);
					try {
						if(strResult != null) {
							Date dateResult = dateFormat.parse(strResult);
							Timestamp tsResult = new Timestamp(dateResult.getTime());
							row.add(tsResult);
						} else {
							row.add(null);
						}
					} catch(ParseException pe) {
						throw new ConnectorException(pe, "Timestamp could not be parsed. Please check to ensure the "
								+ " Format field for attribute "
								+ modelAttrName + " is configured using SimpleDateFormat conventions.");
					}

				}else	if(modelAttrClass.equals(Class.forName(Long.class.getName()))) {
					try {
						//	Throw an exception if class cast fails.
						if(strResult != null) {
							Long longResult = new Long(strResult);
							row.add(longResult);
						} else {
							row.add(null);
						}
					} catch(NumberFormatException nfe) {
						logger.logWarning(nfe.getMessage() + ":Element " + modelAttrName + " is typed as long, " +
								"but it's value (" + strResult + ") cannot be converted from string " +
								"to Long. Defaulted to 0");
						row.add(new Long(0));
				    }

				//	TODO: Extend support for more types in the future.
				// Specifically, add support for byte arrays, since that's actually supported
				// in the underlying data source.

				}
				else {
					throw new ConnectorException("Base type " + modelAttrClass.toString()
							+ " is not supported in the LDAP connector. "
							+ " Please modify the base model to use a supported type.");
				}
			} catch(ClassNotFoundException cne) {
				throw new ConnectorException(cne, "Supported class not found.");
			}
		}
	}


	/**
	 * Active Directory and OpenLDAP supports PagedResultsControls, so I left
	 * this method in here in case we decide to extend support for this control
	 * in the future.
	 */
	private void setADRequestControls(int maxBatchSize) {
		try {
			ldapCtx.setRequestControls(new Control[] { new PagedResultsControl(
					maxBatchSize, Control.CRITICAL) });
		} catch (NamingException ne) {
			logger.logError("Failed to set page size for LDAP results. Please ensure that paged results controls are supported by the LDAP server implementation.");
			ne.printStackTrace();
		} catch (IOException ioe) {
			logger.logError("IO Exception while setting paged results control.");
			ioe.printStackTrace();
		}
	}
}