#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <errno.h>
#include <pthread.h>
#define NUMTHRDS 4
pthread_t callThd[NUMTHRDS];
pthread_mutex_t mutexsum;
char* stradd(const char* a, const char* b){
size_t len = strlen(a) + strlen(b);
char *ret = (char*)malloc(len * sizeof(char) + 1);
*ret = '\0';
return strcat(strcat(ret, a) ,b);
}
void lower_string(char *string)
{
while(*string)
{
if ( *string >= 'A' && *string <= 'Z' )
{
*string = *string + 32;
}
string++;
}
}
void *removeNoise(char *arra1[14284][128], char *noiseList1[15][128]) {
int i, j;
/* mark for removal */
for( i=0; arra1[i][0] != 0x00; i++ ) {
for( j=0; noiseList1[j][0] != 0x00; j++ ) {
if(strcmp(arra1[i], noiseList1[j]) == 0) {
pthread_mutex_lock (&mutexsum);
arra1[i][0]=0x1A; /* ASCII ctrl char for substitute - chosen
arbitrarily */
pthread_mutex_unlock (&mutexsum);
}
}
}
/* one pass to remove */
for( i=0, j=0; arra1[i][0] != 0x00; i++,j++ ) {
while( arra1[i][0] == 0x1A )
i++;
if(i!=j) {
pthread_mutex_lock (&mutexsum);
strcpy(arra1[j],arra1[i]);
pthread_mutex_unlock (&mutexsum);
}
}
pthread_mutex_lock (&mutexsum);
strcpy(arra1[j],arra1[i]);
pthread_mutex_unlock (&mutexsum);
pthread_exit((void*) 0);
}
void *substitution(char *arra1[14284][128], char *originalList[10][128],
char *replacementList[10][128], int size) {
int i, j;
for(i = 0; i < size; i++) {
for(j = 0; j < 10; j++) {
if(strcmp(arra1[i], originalList[j]) == 0) {
pthread_mutex_lock (&mutexsum);
strcpy(arra1[i], replacementList[j]);
pthread_mutex_unlock (&mutexsum);
}
}
}
pthread_exit((void*) 0);
}
void *findFreq(char *arra1[14284][128], int freq[14284], int size) {
int i, j, count;
for(i=0; i<size; i++)
{
/* Initially initialize frequencies to -1 */
freq[i] = -1;
}
for(i=0; i<size; i++)
{
count = 1;
for(j=i+1; j<size; j++)
{
/* If duplicate element is found */
if(strcmp(arra1[i], arra1[j]) == 0)
{
count++;
pthread_mutex_lock (&mutexsum);
/* Make sure not to count frequency of same element again */
freq[j] = 0;
pthread_mutex_unlock (&mutexsum);
}
}
/* If frequency of current element is not counted */
if(freq[i] != 0)
{
pthread_mutex_lock (&mutexsum);
freq[i] = count;
pthread_mutex_unlock (&mutexsum);
}
}
pthread_exit((void*) 0);
/*printf("\nFrequency of all elements of array : \n");
for(i=0; i<size; i++)
{
if(freq[i] != 0)
{
printf("%s occurs %d times\n", arra1[i], freq[i]);
}
}*/
}
void *findCommon(char *arra1[14284][128], int freq[14284], int size) {
int i, j;
int common = 13;
int max, temp;
char tempArr[128][128];
for (i = 0; i < common; i++)
{
pthread_mutex_lock (&mutexsum);
max = i;
// Find next max index
for (j = i+1; j < size; j++)
{
if (freq[j] > freq[max]) {
max = j;
}
}
// Swap numbers in input array
tempArr[0][0] = arra1[i][0];
arra1[i][0] = arra1[max][0];
arra1[max][0] = tempArr[0][0];
// Swap indexes in tracking array
temp = freq[i];
freq[i] = freq[max];
freq[max] = temp;
pthread_mutex_unlock (&mutexsum);
}
for (i = 0; i < common; i++) {
printf("%d -> %s\n", freq[i], arra1[i]);
}
pthread_exit((void*) 0);
}
/* Read in all files from a folder */
int main(int argc, char **argv)
{
char *fileArray[78];
int i, j;
DIR *d;
struct dirent *dir;
char arra[128][128];
char *arra1[14284][128];
char line[1024];
char line1[1024];
char line2[1024];
char noiseList[128][128];
char *noiseList1[15][128];
char conceptList[128][128];
char *conceptList1[20][128];
char *originalList[10][128];
char *replacementList[10][128];
char *token;
char *token1;
char *path = "./alerts2013/2013/";
char *path1 = "./Noise_and_Concepts/";
char *fileName;
char *fileName1;
char *fileName2;
int freq[14284];
char seps[] = " ,\t\n";
FILE *myfile;
FILE *noise;
FILE *concept;
int size = 0;
int replaceSize = 0;
d = opendir("./alerts2013/2013");
fileName1 = stradd(path1, "noise.txt");
fileName2 = stradd(path1, "concepts.txt");
//printf("%s\n", fileName1);
noise = fopen(fileName1, "r");
if (noise == 0)
{
printf("can not open file \n");
exit(1);
}
int a, b;
for(a = 0; a < 128; a++) {
line[a] = '\0';
line2[a] = '\0';
}
for(a = 0; a < 128; a++) {
for(b = 0; b < 128; b++) {
noiseList[a][b] = '\0';
noiseList1[a][b] = '\0';
arra[a][b] = '\0';
arra1[a][b] = '\0';
conceptList[a][b] = '\0';
conceptList1[a][b] = '\0';
originalList[a][b] = '\0';
replacementList[a][b] = '\0';
}
}
i = 0;
j = 0;
int k = 0;
int l = 0;
int m = 0;
int n = 0;
int q = 0;
int r = 0;
while(fgets(line, sizeof(line), noise) != NULL) {
strcpy(noiseList[k], line);
//printf("%s", noiseList[k]);
token = strtok(noiseList[k], seps);
while(token != NULL )
{
/* While there are tokens in "string" */
//printf("%s\n", token);
//printf("array ----> %s\n", token);
lower_string(token);
strcpy(noiseList1[n], token);
n++;
/* Get next token: */
token = strtok( NULL, seps );
}
k++;
}
concept = fopen(fileName2, "r");
if (concept == 0)
{
printf("can not open file \n");
exit(1);
}
while(fgets(line2, sizeof(line2), concept) != NULL) {
strcpy(conceptList[q], line2);
//printf("%s", noiseList[q]);
token = strtok(conceptList[q], seps);
while(token != NULL )
{
/* While there are tokens in "string" */
//printf("%s\n", token);
//printf("array ----> %s\n", token);
lower_string(token);
strcpy(conceptList1[r], token);
r++;
/* Get next token: */
token = strtok( NULL, seps );
}
q++;
}
for(i = 0; i < 20; i++) {
if(i == 0) {
strcpy(originalList[0], conceptList1[0]);
}
if(i % 2 == 0) {
strcpy(originalList[i/2], conceptList1[i]);
}
else {
strcpy(replacementList[replaceSize], conceptList1[i]);
replaceSize++;
}
}
if (d)
{
while ((dir = readdir(d)) != NULL)
{
fileArray[i] = dir->d_name;
//printf("%s\n", fileArray[i]);
fileName = stradd(path, dir->d_name);
//printf("%s\n", fileName);
free(fileName);
myfile = fopen(fileName,"r");
if (myfile == 0)
{
printf("can not open file \n");
exit(1);
}
for(i = 0; i < 128; i++) {
line1[i] = '\0';
}
if(myfile != NULL) {
while(fgets(line1, sizeof(line1), myfile) != NULL) {
strcpy(arra[l], line1);
//printf("Tokens:\n" );
/* Establish string and get the first token: */
token = strtok(arra[l], seps);
while(token != NULL )
{
/* While there are tokens in "string" */
//printf("%s\n", token);
//printf("array ----> %s\n", token);
lower_string(token);
strcpy(arra1[m], token);
//printf("Arra1: %s\n", arra1[m]);
size++;
m++;
/* Get next token: */
token = strtok( NULL, seps );
}
//printf("array ----> %s ", &arra[i]);
i++;
}
}
fclose(myfile);
i++;
}
closedir(d);
}
pthread_attr_t attr;
void *status;
pthread_mutex_init(&mutexsum, NULL);
/* Create threads to perform the dotproduct */
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for(i=0;i<NUMTHRDS;i++)
{
/* Each thread works on a different set of data.
* The offset is specified by 'i'. The size of
* the data for each thread is indicated by VECLEN.
*/
pthread_create(&callThd[i], &attr, removeNoise(arra1, noiseList1),
(void *)i);
}
pthread_attr_destroy(&attr);
/* Wait on the other threads */
for(i=0;i<NUMTHRDS;i++) {
pthread_join(callThd[i], &status);
}
for(i=0;i<NUMTHRDS;i++)
{
/* Each thread works on a different set of data.
* The offset is specified by 'i'. The size of
* the data for each thread is indicated by VECLEN.
*/
pthread_create(&callThd[i], &attr, substitution(arra1,
originalList, replacementList, size), (void *)i);
}
pthread_attr_destroy(&attr);
/* Wait on the other threads */
for(i=0;i<NUMTHRDS;i++) {
pthread_join(callThd[i], &status);
}
for(i=0;i<NUMTHRDS;i++)
{
/* Each thread works on a different set of data.
* The offset is specified by 'i'. The size of
* the data for each thread is indicated by VECLEN.
*/
pthread_create(&callThd[i], &attr, findFreq(arra1, freq, size),
(void *)i);
}
pthread_attr_destroy(&attr);
/* Wait on the other threads */
for(i=0;i<NUMTHRDS;i++) {
pthread_join(callThd[i], &status);
}
for(i=0;i<NUMTHRDS;i++)
{
/* Each thread works on a different set of data.
* The offset is specified by 'i'. The size of
* the data for each thread is indicated by VECLEN.
*/
pthread_create(&callThd[i], &attr, findCommon(arra1, freq, size),
(void *)i);
}
pthread_attr_destroy(&attr);
/* Wait on the other threads */
for(i=0;i<NUMTHRDS;i++) {
pthread_join(callThd[i], &status);
}
/*int p;
int w;
printf("%d\n", size);
for(p = 0; p < 10; p++) {
printf("%s\n", replacementList[p]);
}
for(w = 0; w < size; w++) {
printf("Arr1 (final): %s\n", arra1[w]);
}*/
FILE * Output;
Output = fopen("data.txt", "w");
for(i = 2; i < 12; i++) {
fprintf(Output, "%d -> %s", freq[i], arra1[i]);
fprintf(Output, " ");
}
fclose(Output);
fclose(noise);
fclose(concept);
pthread_mutex_destroy(&mutexsum);
pthread_exit(NULL);
return(0);
}
