ms sql query on count occurence of words in text c

Question

I have a table called webqueries with a column named qQuestion of data type text(sql server 2008). I want to create a count on words used in qQuestion (excluding 'and', 'is' etc). My goal is to see how many times a person has asked a question about a specific product.

Answer 1

You could create a table-valued function to parse words and join it to your query against qQuestion. In your schema, I recommend using varchar(8000) or varchar(max) instead of text. Meanwhile, the following should get you started:

create function [dbo].[fnParseWords](@str varchar(max), @delimiter varchar(30)='%[^a-zA-Z0-9\_]%')
returns @result table(word varchar(max))
begin
    if left(@delimiter,1)<>'%' set @delimiter='%'+@delimiter;
    if right(@delimiter,1)<>'%' set @delimiter+='%';
    set @str=rtrim(@str);
    declare @pi int=PATINDEX(@delimiter,@str);

    while @pi>0 begin
        insert into @result select LEFT(@str,@pi-1) where @pi>1;
        set @str=RIGHT(@str,len(@str)-@pi);
        set @pi=PATINDEX(@delimiter,@str);
    end

    insert into @result select @str where LEN(@str)>0;
    return;
end
go

select COUNT(*)
from webqueries q
cross apply dbo.fnParseWords(cast(q.qQuestion as varchar(max)),default) pw
where pw.word not in ('and','is','a','the'/* plus whatever else you need to exclude */)