138private void AddWebPage(Uri objBaseUri, string strNewUri)
139{
140	string strURL = StrUtil.LeftIndexOf(strNewUri, "#");
141	Uri objUri = new Uri(objBaseUri, strURL);
142	if (!ValidPage(objUri.LocalPath ) || mHWebPages.Contains(objUri))
143		return;
144	if (objUri.AbsoluteUri.StartsWith(objBaseUri.AbsoluteUri))
145	{
146	  WebPageState objState = new WebPageState(objUri);
147	  objState.ID = mintIDs;	//Assign a unique ID for the cache
148	  mintIDs++;				//Bump up count
149	  mQWebPagesPending.Enqueue(objState);
150	  mHWebPages.Add(objUri, objState);
151	}
152	return;
153}


171public bool Process(WebPageState objState)
172{
173	objState.ProcessStarted = true;
174	objState.ProcessSuccessful = false;
175	try
176	{
177	  WebRequest objReq = WebRequest.Create(objState.Uri);
178	  WebResponse objRes = null;
179	  try
180	  {
181	    objRes = objReq.GetResponse();
182	    if (objRes is HttpWebResponse)
183	    {
184	      objState.StatusCode =
		((HttpWebResponse)objRes).StatusCode.ToString();
185	      objState.StatusDescription =
		((HttpWebResponse)objRes).StatusDescription;
186	    }
187	    if (objRes is FileWebResponse)
188	    {
189	      objState.StatusCode        = "OK";
190	      objState.StatusDescription = "OK";
191	    }
192	    if (objState.StatusCode.Equals("OK"))
193	    {
194	      StreamReader objSR = new
		StreamReader(objRes.GetResponseStream());
195	      objState.Content = objSR.ReadToEnd();
196	    }
197	    objState.ProcessSuccessful = true;
198	  }
199	  catch( Exception ex )
200	  {
201	    HandleException( ex, objState );
202	  }
203	  finally
204	  {
205	    if (objRes != null)
206	    objRes.Close( );
207	  }
208	}
209	catch (Exception ex)
210	{
211	  Console.WriteLine(ex.ToString());
212	}
213	return objState.ProcessSuccessful;
214}


261public void HandleLinks(WebPageState objState)
262{
263	Match m = RegExUtil.GetMatchRegEx(
		RegExUtil.RegularExpression.UrlExtractor, objState.Content);
264	while( m.Success )
265	{
266	  AddWebPage(objState.Uri, m.Groups["url"].ToString());
267	  m = m.NextMatch();
268	}
269}


272//Parses and indexes a webpage
273public void AddWebPageToIndex(WebPageState objState)
274{
275	if (objState.Content != null)
276	{
277    Document objDoc = new Document();
278    objDoc.Add(Field.UnIndexed("id", objState.ID.ToString()));
279    objDoc.Add(Field.Text("title", GetTitle(objState.Content)));
280    objDoc.Add(Field.Keyword("path", objState.Uri.AbsoluteUri));
281    objDoc.Add(Field.UnStored("text", objState.Content));
282    mobjWriter.AddDocument(objDoc);
283    CacheFile(objState);
284	}
285}