csharp/Adoxio/xRM-Portals-Community-Edition/Framework/Adxstudio.Xrm/Search/Index/CrmEntityIndexBuilder.cs

CrmEntityIndexBuilder.cs
/*
  Copyright (c) Microsoft Corporation. All rights reserved.
  Licensed under the MIT License. See License.txt in the project root for license information.
*/

using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Collections.Generic;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Microsoft.Practices.TransientFaultHandling;
using Fetch = Adxstudio.Xrm.Services.Query;
using Adxstudio.Xrm.Cms;
using Adxstudio.Xrm.Configuration;
using Adxstudio.Xrm.Diagnostics.Trace;

namespace Adxstudio.Xrm.Search.Index
{
	public clast CrmEnsatyIndexBuilder : ICrmEnsatyIndexBuilder, ICrmEnsatyIndexUpdater
	{
		private readonly ICrmEnsatyIndex _index;

		public CrmEnsatyIndexBuilder(ICrmEnsatyIndex index)
		{
			if (index == null)
			{
				throw new ArgumentNullException("index");
			}

			_index = index;
		}

		public void BuildIndex()
		{
			var timer = Stopwatch.StartNew();

			ADXTrace.Instance.TraceInfo(TraceCategory.Application, "Start");

			var indexers = _index.GetIndexers();

			ADXTrace.Instance.TraceInfo(TraceCategory.Application, "Retrieving index docameents");

			var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents());

			UsingWriter(MethodBase.GetCurrentMethod().Name, true, true, writer =>
			{
				foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
				{
					writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
				}
			});

			timer.Stop();

			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("End. Elapsed time: {0}", timer.ElapsedMilliseconds));
		}

		public void Dispose() { }

		public void DeleteEnsaty(string ensatyLogicalName, Guid id)
		{
			var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();

			if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
			{
				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));

				return;
			}

			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Deleting index for EnsatyLogicalName: {0}, Guid: {1} ", EnsatyNamePrivacy.GetEnsatyName(ensatyLogicalName), id));

			UsingWriter(MethodBase.GetCurrentMethod().Name, false, false, writer => writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, id)));
		}

		public void DeleteEnsatySet(string ensatyLogicalName)
		{
			var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();

			if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
			{
				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));

				return;
			}

			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Deleting index set for EnsatyLogicalName: {0}", EnsatyNamePrivacy.GetEnsatyName(ensatyLogicalName)));

			UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer => writer.DeleteDocameents(new Term(_index.LogicalNameFieldName, ensatyLogicalName)));
		}

		public void UpdateEnsaty(string ensatyLogicalName, Guid id)
		{
			var indexers = _index.GetIndexers(ensatyLogicalName, id).ToArray();

			if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
			{
				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));

				return;
			}

			var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents()).ToArray();

			UsingWriter(MethodBase.GetCurrentMethod().Name, false, false, writer =>
			{
				writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, id));

				foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
				{
					writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
				}
			});
		}

		public void UpdateEnsatySet(string ensatyLogicalName)
		{
			var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();

			if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
			{
				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));

				return;
			}

			var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents());

			UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
			{
				writer.DeleteDocameents(new Term(_index.LogicalNameFieldName, ensatyLogicalName));

				foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
				{
					writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
				}
			});
		}

		public void UpdateEnsatySet(string ensatyLogicalName, string ensatyAttribute, List ensatyIds)
		{
			var filter = new Fetch.Filter
			{
				Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
				Conditions = new List()
					{
						new Fetch.Condition
						{
							Attribute = ensatyAttribute,
							Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
							Values = ensatyIds.Cast().ToList()
						},
					}
			};

			var ensatyIndexers = _index.GetIndexers(ensatyLogicalName, filters: new List { filter });
			UpdateWithIndexers(ensatyLogicalName, ensatyIndexers);
		}

		public void UpdateCmsEnsatyTree(string ensatyLogicalName, Guid rootEnsatyId, int? lcid = null)
		{
			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Begin updating Cms Ensaty Tree for logical name: {0}, rootEnsatyId: {1}", ensatyLogicalName, rootEnsatyId));
			var timer = Stopwatch.StartNew();

			if (ensatyLogicalName == "adx_webpage")
			{
				IContentMapProvider contentMapProvider = AdxstudioCrmConfigurationManager.CreateContentMapProvider();
				Guid[] descendantLocalizedWebPagesGuids = CmsIndexHelper.GetDescendantLocalizedWebpagesForWebpage(contentMapProvider, rootEnsatyId, lcid).ToArray();
				Guid[] descendantRootWebPagesGuids = CmsIndexHelper.GetDescendantRootWebpagesForWebpage(contentMapProvider, rootEnsatyId).ToArray();

				// -------------------- WEB PAGES ------------------------------
				if (descendantLocalizedWebPagesGuids.Any())
				{
					var localizedWebPagesUnderTargetWebPageFilter = new Fetch.Filter
					{
						Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
						Conditions = new List()
						{
							new Fetch.Condition
							{
								Attribute = "adx_webpageid",
								Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
								Values = descendantLocalizedWebPagesGuids.Cast().ToList()
							},
						}
					};

					var webPageIndexers = _index.GetIndexers("adx_webpage", filters: new List { localizedWebPagesUnderTargetWebPageFilter });

					UpdateWithIndexers("adx_webpage", webPageIndexers);
				}

				// -------------------- FORUMS ------------------------------
				if (descendantRootWebPagesGuids.Any())
				{
					var rootWebPagesUnderTargetWebPageFilter = new Fetch.Filter
					{
						Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
						Conditions = new List()
						{
							new Fetch.Condition
							{
								Attribute = "adx_webpageid",
								Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
								Values = descendantRootWebPagesGuids.Cast().ToList()
							},
						}
					};

					var forumBlogToParentPageLink = new Fetch.Link
					{
						Name = "adx_webpage",
						FromAttribute = "adx_webpageid",
						ToAttribute = "adx_parentpageid",
						Filters = new List()
						{
							rootWebPagesUnderTargetWebPageFilter
						}
					};

					Fetch.Link languageFilter = null;

					if (lcid.HasValue)
					{
						languageFilter = new Fetch.Link
						{
							Name = "adx_websitelanguage",
							FromAttribute = "adx_websitelanguageid",
							ToAttribute = "adx_websitelanguageid",
							Type = Microsoft.Xrm.Sdk.Query.JoinOperator.Inner,
							Alias = "websitelangforupdatefilter",
							Links = new List()
							{
								new Fetch.Link
								{
									Name = "adx_portallanguage",
									FromAttribute = "adx_portallanguageid",
									ToAttribute = "adx_portallanguageid",
									Type = Microsoft.Xrm.Sdk.Query.JoinOperator.Inner,
									Alias = "portallangforupdatefilter",
									Filters = new List()
									{
										new Fetch.Filter
										{
											Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
											Conditions = new List()
											{
												new Fetch.Condition
												{
													Attribute = "adx_lcid",
													Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
													Value = lcid.Value
												}
											}
										}
									}
								}
							}
						};
					}

					var forumBlogLinks = new List() { forumBlogToParentPageLink };
					if (languageFilter != null)
					{
						forumBlogLinks.Add(languageFilter);
					}

					var forumIndexers = _index.GetIndexers("adx_communityforum", links: forumBlogLinks);
					UpdateWithIndexers("adx_communityforum", forumIndexers);

					var forumThreadForumLinks = new List() { forumBlogToParentPageLink };
					if (languageFilter != null)
					{
						forumThreadForumLinks.Add(languageFilter);
					}

					var forumThreadToParentPageLink = new Fetch.Link
					{
						Name = "adx_communityforum",
						FromAttribute = "adx_communityforumid",
						ToAttribute = "adx_forumid",
						Links = forumThreadForumLinks
					};

					var forumThreadIndexers = _index.GetIndexers("adx_communityforumthread",
						links: new List() { forumThreadToParentPageLink });
					UpdateWithIndexers("adx_communityforumthread", forumThreadIndexers);

					var forumPostToParentPageLink = new Fetch.Link
					{
						Name = "adx_communityforumthread",
						FromAttribute = "adx_communityforumthreadid",
						ToAttribute = "adx_forumthreadid",
						Alias = "adx_communityforumpost_communityforumthread",
						Links = new List()
						{
							forumThreadToParentPageLink
						}
					};

					var forumPostIndexers = _index.GetIndexers("adx_communityforumpost",
						links: new List() { forumPostToParentPageLink });
					UpdateWithIndexers("adx_communityforumpost", forumPostIndexers);

					// -------------------- BLOGS ------------------------------
					var blogIndexers = _index.GetIndexers("adx_blog", links: forumBlogLinks);
					UpdateWithIndexers("adx_blog", blogIndexers);

					var blogPostBlogLinks = new List() { forumBlogToParentPageLink };
					if (languageFilter != null)
					{
						blogPostBlogLinks.Add(languageFilter);
					}

					var blogPostParentPageLink = new Fetch.Link
					{
						Name = "adx_blog",
						FromAttribute = "adx_blogid",
						ToAttribute = "adx_blogid",
						Alias = "adx_blog_blogpost",
						Links = blogPostBlogLinks
					};

					var blogPostIndexers = _index.GetIndexers("adx_blogpost", links: new List { blogPostParentPageLink });
					UpdateWithIndexers("adx_blogpost", blogPostIndexers);
				}
			}
			else if (ensatyLogicalName == "adx_communityforum")
			{
				UpdateEnsaty("adx_communityforum", rootEnsatyId);

				var inForumFilterForThread = new Fetch.Filter
				{
					Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
					Conditions = new List()
					{
						new Fetch.Condition
						{
							Attribute = "adx_forumid",
							Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
							Value = rootEnsatyId
						}
					}
				};

				var forumThreadIndexers = _index.GetIndexers("adx_communityforumthread", filters: new List { inForumFilterForThread });
				UpdateWithIndexers("adx_communityforumthread", forumThreadIndexers);

				var inForumFilterForPost = new Fetch.Link
				{
					Name = "adx_communityforumthread",
					FromAttribute = "adx_communityforumthreadid",
					ToAttribute = "adx_forumthreadid",
					Alias = "adx_communityforumpost_communityforumthread",
					Filters = new List()
					{
					   inForumFilterForThread
					}
				};

				var forumPostIndexers = _index.GetIndexers("adx_communityforumpost", links: new List { inForumFilterForPost });
				UpdateWithIndexers("adx_communityforumpost", forumPostIndexers);
			}
			else if (ensatyLogicalName == "adx_ideaforum")
			{
				UpdateEnsaty("adx_ideaforum", rootEnsatyId);

				var inIdeaForumFilter = new Fetch.Filter
				{
					Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
					Conditions = new List()
					{
						new Fetch.Condition
						{
							Attribute = "adx_ideaforumid",
							Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
							Value = rootEnsatyId
						}
					}
				};

				var ideaIndexers = _index.GetIndexers("adx_idea", filters: new List { inIdeaForumFilter });
				UpdateWithIndexers("adx_idea", ideaIndexers);
			}

			timer.Stop();
			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Cms Ensaty Tree updated for logical name: {0}, rootEnsatyId: {1}, timespan: {2}", ensatyLogicalName, rootEnsatyId, timer.ElapsedMilliseconds));
		}

		private void UpdateWithIndexers(string ensatyLogicalName, IEnumerable indexers)
		{
			if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
			{
				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));

				return;
			}

			var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents()).ToArray();

			UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
			{
				foreach (var ensatyDoc in ensatyIndexDocameents)
				{
					writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, ensatyDoc.PrimaryKey));
				}
			});

			int currentIndex = 0;
			while (currentIndex < ensatyIndexDocameents.Length)
			{
				UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
				{
					var stopwatch = new Stopwatch();
					stopwatch.Start();

					for (; currentIndex < ensatyIndexDocameents.Length; currentIndex++)
					{
						writer.AddDocameent(ensatyIndexDocameents[currentIndex].Docameent, ensatyIndexDocameents[currentIndex].astyzer);

						// We've held onto the write lock too long, there might be other updates waiting on us.
						// Release the lock so they don't time out, then re-enter the queue for the write lock.
						if (stopwatch.Elapsed.TotalSeconds > 10)
						{
							// break;
						}
					}
				});
			}
		}

		public override string ToString()
		{
			return _index.Directory.ToString();
		}

		protected virtual void UsingWriter(string description, bool create, bool optimize, Action action)
		{
			ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Start", description));

			var stopwatch = new Stopwatch();

			stopwatch.Start();

			try
			{
				var retryPolicy = new RetryPolicy(new LockObtainTransientErrorDetectionStrategy(), 25, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(2));

				using (var writer = retryPolicy.ExecuteAction(() => new IndexWriter(_index.Directory, _index.astyzer, create, IndexWriter.MaxFieldLength.UNLIMITED)))
				{
					try
					{
						ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Acquired write lock, writing", description));

						action(writer);

						if (optimize)
						{
							ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Optimizing index", description));

							writer.Optimize();
						}
					}
					catch (Exception e)
					{
						ADXTrace.Instance.TraceWarning(TraceCategory.Application, string.Format("{0}: Error during index write: rollback, rethrow: {1}", description, e));

						writer.Rollback();

						throw;
					}
				}

				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Index writer closed", description));
			}
			catch (Exception e)
			{
				SearchEventSource.Log.WriteError(e);

				throw;
			}
			finally
			{
				stopwatch.Stop();

				ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: End (Elapsed time: {1})", description, stopwatch.Elapsed));
			}
		}

		private static Query GetEnsatyQuery(ICrmEnsatyIndex index, string ensatyLogicalName, Guid id)
		{
			return new BooleanQuery
			{
				{ new TermQuery(new Term(index.LogicalNameFieldName, ensatyLogicalName)), Occur.MUST },
				{ new TermQuery(new Term(index.PrimaryKeyFieldName, id.ToString())), Occur.MUST }
			};
		}

		private clast LockObtainTransientErrorDetectionStrategy : ITransientErrorDetectionStrategy
		{
			public bool IsTransient(Exception e)
			{
				return (e is LockObtainFailedException
					|| e is IOException
					|| e is UnauthorizedAccessException);
			}
		}
	}
}