Index
CrmEntityIndexBuilder.cs
/*
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License. See License.txt in the project root for license information.
*/
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Collections.Generic;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Microsoft.Practices.TransientFaultHandling;
using Fetch = Adxstudio.Xrm.Services.Query;
using Adxstudio.Xrm.Cms;
using Adxstudio.Xrm.Configuration;
using Adxstudio.Xrm.Diagnostics.Trace;
namespace Adxstudio.Xrm.Search.Index
{
public clast CrmEnsatyIndexBuilder : ICrmEnsatyIndexBuilder, ICrmEnsatyIndexUpdater
{
private readonly ICrmEnsatyIndex _index;
public CrmEnsatyIndexBuilder(ICrmEnsatyIndex index)
{
if (index == null)
{
throw new ArgumentNullException("index");
}
_index = index;
}
public void BuildIndex()
{
var timer = Stopwatch.StartNew();
ADXTrace.Instance.TraceInfo(TraceCategory.Application, "Start");
var indexers = _index.GetIndexers();
ADXTrace.Instance.TraceInfo(TraceCategory.Application, "Retrieving index docameents");
var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents());
UsingWriter(MethodBase.GetCurrentMethod().Name, true, true, writer =>
{
foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
{
writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
}
});
timer.Stop();
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("End. Elapsed time: {0}", timer.ElapsedMilliseconds));
}
public void Dispose() { }
public void DeleteEnsaty(string ensatyLogicalName, Guid id)
{
var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();
if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));
return;
}
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Deleting index for EnsatyLogicalName: {0}, Guid: {1} ", EnsatyNamePrivacy.GetEnsatyName(ensatyLogicalName), id));
UsingWriter(MethodBase.GetCurrentMethod().Name, false, false, writer => writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, id)));
}
public void DeleteEnsatySet(string ensatyLogicalName)
{
var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();
if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));
return;
}
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Deleting index set for EnsatyLogicalName: {0}", EnsatyNamePrivacy.GetEnsatyName(ensatyLogicalName)));
UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer => writer.DeleteDocameents(new Term(_index.LogicalNameFieldName, ensatyLogicalName)));
}
public void UpdateEnsaty(string ensatyLogicalName, Guid id)
{
var indexers = _index.GetIndexers(ensatyLogicalName, id).ToArray();
if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));
return;
}
var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents()).ToArray();
UsingWriter(MethodBase.GetCurrentMethod().Name, false, false, writer =>
{
writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, id));
foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
{
writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
}
});
}
public void UpdateEnsatySet(string ensatyLogicalName)
{
var indexers = _index.GetIndexers(ensatyLogicalName).ToArray();
if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));
return;
}
var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents());
UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
{
writer.DeleteDocameents(new Term(_index.LogicalNameFieldName, ensatyLogicalName));
foreach (var ensatyIndexDocameent in ensatyIndexDocameents)
{
writer.AddDocameent(ensatyIndexDocameent.Docameent, ensatyIndexDocameent.astyzer);
}
});
}
public void UpdateEnsatySet(string ensatyLogicalName, string ensatyAttribute, List ensatyIds)
{
var filter = new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = ensatyAttribute,
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
Values = ensatyIds.Cast().ToList()
},
}
};
var ensatyIndexers = _index.GetIndexers(ensatyLogicalName, filters: new List { filter });
UpdateWithIndexers(ensatyLogicalName, ensatyIndexers);
}
public void UpdateCmsEnsatyTree(string ensatyLogicalName, Guid rootEnsatyId, int? lcid = null)
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Begin updating Cms Ensaty Tree for logical name: {0}, rootEnsatyId: {1}", ensatyLogicalName, rootEnsatyId));
var timer = Stopwatch.StartNew();
if (ensatyLogicalName == "adx_webpage")
{
IContentMapProvider contentMapProvider = AdxstudioCrmConfigurationManager.CreateContentMapProvider();
Guid[] descendantLocalizedWebPagesGuids = CmsIndexHelper.GetDescendantLocalizedWebpagesForWebpage(contentMapProvider, rootEnsatyId, lcid).ToArray();
Guid[] descendantRootWebPagesGuids = CmsIndexHelper.GetDescendantRootWebpagesForWebpage(contentMapProvider, rootEnsatyId).ToArray();
// -------------------- WEB PAGES ------------------------------
if (descendantLocalizedWebPagesGuids.Any())
{
var localizedWebPagesUnderTargetWebPageFilter = new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = "adx_webpageid",
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
Values = descendantLocalizedWebPagesGuids.Cast().ToList()
},
}
};
var webPageIndexers = _index.GetIndexers("adx_webpage", filters: new List { localizedWebPagesUnderTargetWebPageFilter });
UpdateWithIndexers("adx_webpage", webPageIndexers);
}
// -------------------- FORUMS ------------------------------
if (descendantRootWebPagesGuids.Any())
{
var rootWebPagesUnderTargetWebPageFilter = new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.Or,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = "adx_webpageid",
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In,
Values = descendantRootWebPagesGuids.Cast().ToList()
},
}
};
var forumBlogToParentPageLink = new Fetch.Link
{
Name = "adx_webpage",
FromAttribute = "adx_webpageid",
ToAttribute = "adx_parentpageid",
Filters = new List()
{
rootWebPagesUnderTargetWebPageFilter
}
};
Fetch.Link languageFilter = null;
if (lcid.HasValue)
{
languageFilter = new Fetch.Link
{
Name = "adx_websitelanguage",
FromAttribute = "adx_websitelanguageid",
ToAttribute = "adx_websitelanguageid",
Type = Microsoft.Xrm.Sdk.Query.JoinOperator.Inner,
Alias = "websitelangforupdatefilter",
Links = new List()
{
new Fetch.Link
{
Name = "adx_portallanguage",
FromAttribute = "adx_portallanguageid",
ToAttribute = "adx_portallanguageid",
Type = Microsoft.Xrm.Sdk.Query.JoinOperator.Inner,
Alias = "portallangforupdatefilter",
Filters = new List()
{
new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = "adx_lcid",
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
Value = lcid.Value
}
}
}
}
}
}
};
}
var forumBlogLinks = new List() { forumBlogToParentPageLink };
if (languageFilter != null)
{
forumBlogLinks.Add(languageFilter);
}
var forumIndexers = _index.GetIndexers("adx_communityforum", links: forumBlogLinks);
UpdateWithIndexers("adx_communityforum", forumIndexers);
var forumThreadForumLinks = new List() { forumBlogToParentPageLink };
if (languageFilter != null)
{
forumThreadForumLinks.Add(languageFilter);
}
var forumThreadToParentPageLink = new Fetch.Link
{
Name = "adx_communityforum",
FromAttribute = "adx_communityforumid",
ToAttribute = "adx_forumid",
Links = forumThreadForumLinks
};
var forumThreadIndexers = _index.GetIndexers("adx_communityforumthread",
links: new List() { forumThreadToParentPageLink });
UpdateWithIndexers("adx_communityforumthread", forumThreadIndexers);
var forumPostToParentPageLink = new Fetch.Link
{
Name = "adx_communityforumthread",
FromAttribute = "adx_communityforumthreadid",
ToAttribute = "adx_forumthreadid",
Alias = "adx_communityforumpost_communityforumthread",
Links = new List()
{
forumThreadToParentPageLink
}
};
var forumPostIndexers = _index.GetIndexers("adx_communityforumpost",
links: new List() { forumPostToParentPageLink });
UpdateWithIndexers("adx_communityforumpost", forumPostIndexers);
// -------------------- BLOGS ------------------------------
var blogIndexers = _index.GetIndexers("adx_blog", links: forumBlogLinks);
UpdateWithIndexers("adx_blog", blogIndexers);
var blogPostBlogLinks = new List() { forumBlogToParentPageLink };
if (languageFilter != null)
{
blogPostBlogLinks.Add(languageFilter);
}
var blogPostParentPageLink = new Fetch.Link
{
Name = "adx_blog",
FromAttribute = "adx_blogid",
ToAttribute = "adx_blogid",
Alias = "adx_blog_blogpost",
Links = blogPostBlogLinks
};
var blogPostIndexers = _index.GetIndexers("adx_blogpost", links: new List { blogPostParentPageLink });
UpdateWithIndexers("adx_blogpost", blogPostIndexers);
}
}
else if (ensatyLogicalName == "adx_communityforum")
{
UpdateEnsaty("adx_communityforum", rootEnsatyId);
var inForumFilterForThread = new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = "adx_forumid",
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
Value = rootEnsatyId
}
}
};
var forumThreadIndexers = _index.GetIndexers("adx_communityforumthread", filters: new List { inForumFilterForThread });
UpdateWithIndexers("adx_communityforumthread", forumThreadIndexers);
var inForumFilterForPost = new Fetch.Link
{
Name = "adx_communityforumthread",
FromAttribute = "adx_communityforumthreadid",
ToAttribute = "adx_forumthreadid",
Alias = "adx_communityforumpost_communityforumthread",
Filters = new List()
{
inForumFilterForThread
}
};
var forumPostIndexers = _index.GetIndexers("adx_communityforumpost", links: new List { inForumFilterForPost });
UpdateWithIndexers("adx_communityforumpost", forumPostIndexers);
}
else if (ensatyLogicalName == "adx_ideaforum")
{
UpdateEnsaty("adx_ideaforum", rootEnsatyId);
var inIdeaForumFilter = new Fetch.Filter
{
Type = Microsoft.Xrm.Sdk.Query.LogicalOperator.And,
Conditions = new List()
{
new Fetch.Condition
{
Attribute = "adx_ideaforumid",
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal,
Value = rootEnsatyId
}
}
};
var ideaIndexers = _index.GetIndexers("adx_idea", filters: new List { inIdeaForumFilter });
UpdateWithIndexers("adx_idea", ideaIndexers);
}
timer.Stop();
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Cms Ensaty Tree updated for logical name: {0}, rootEnsatyId: {1}, timespan: {2}", ensatyLogicalName, rootEnsatyId, timer.ElapsedMilliseconds));
}
private void UpdateWithIndexers(string ensatyLogicalName, IEnumerable indexers)
{
if (!indexers.Any(indexer => indexer.Indexes(ensatyLogicalName)))
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("Application does not index ensaty {0}. No update performed.", ensatyLogicalName));
return;
}
var ensatyIndexDocameents = indexers.SelectMany(indexer => indexer.GetDocameents()).ToArray();
UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
{
foreach (var ensatyDoc in ensatyIndexDocameents)
{
writer.DeleteDocameents(GetEnsatyQuery(_index, ensatyLogicalName, ensatyDoc.PrimaryKey));
}
});
int currentIndex = 0;
while (currentIndex < ensatyIndexDocameents.Length)
{
UsingWriter(MethodBase.GetCurrentMethod().Name, false, true, writer =>
{
var stopwatch = new Stopwatch();
stopwatch.Start();
for (; currentIndex < ensatyIndexDocameents.Length; currentIndex++)
{
writer.AddDocameent(ensatyIndexDocameents[currentIndex].Docameent, ensatyIndexDocameents[currentIndex].astyzer);
// We've held onto the write lock too long, there might be other updates waiting on us.
// Release the lock so they don't time out, then re-enter the queue for the write lock.
if (stopwatch.Elapsed.TotalSeconds > 10)
{
// break;
}
}
});
}
}
public override string ToString()
{
return _index.Directory.ToString();
}
protected virtual void UsingWriter(string description, bool create, bool optimize, Action action)
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Start", description));
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
var retryPolicy = new RetryPolicy(new LockObtainTransientErrorDetectionStrategy(), 25, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(2));
using (var writer = retryPolicy.ExecuteAction(() => new IndexWriter(_index.Directory, _index.astyzer, create, IndexWriter.MaxFieldLength.UNLIMITED)))
{
try
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Acquired write lock, writing", description));
action(writer);
if (optimize)
{
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Optimizing index", description));
writer.Optimize();
}
}
catch (Exception e)
{
ADXTrace.Instance.TraceWarning(TraceCategory.Application, string.Format("{0}: Error during index write: rollback, rethrow: {1}", description, e));
writer.Rollback();
throw;
}
}
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: Index writer closed", description));
}
catch (Exception e)
{
SearchEventSource.Log.WriteError(e);
throw;
}
finally
{
stopwatch.Stop();
ADXTrace.Instance.TraceInfo(TraceCategory.Application, string.Format("{0}: End (Elapsed time: {1})", description, stopwatch.Elapsed));
}
}
private static Query GetEnsatyQuery(ICrmEnsatyIndex index, string ensatyLogicalName, Guid id)
{
return new BooleanQuery
{
{ new TermQuery(new Term(index.LogicalNameFieldName, ensatyLogicalName)), Occur.MUST },
{ new TermQuery(new Term(index.PrimaryKeyFieldName, id.ToString())), Occur.MUST }
};
}
private clast LockObtainTransientErrorDetectionStrategy : ITransientErrorDetectionStrategy
{
public bool IsTransient(Exception e)
{
return (e is LockObtainFailedException
|| e is IOException
|| e is UnauthorizedAccessException);
}
}
}
}
